From 9230e2b8888830d681d429d5c6a1761bfb1dd4ab Mon Sep 17 00:00:00 2001 From: Steven Hartland Date: Mon, 5 Dec 2022 23:15:21 +0000 Subject: [PATCH] fix: tester (#478) Fix the way tester works, move it into the tools directory and clean up how it processes files including adding more data to the summary about what failed and why. --- .gitignore | 12 +- test/tester.go | 227 ---------------------------- tools/tester/main.go | 341 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 348 insertions(+), 232 deletions(-) delete mode 100644 test/tester.go create mode 100644 tools/tester/main.go diff --git a/.gitignore b/.gitignore index 0dfceb5..ea050b8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,10 @@ -/.test -/otto/otto -/otto/otto-* -/test/test-*.js -/test/tester +.test +otto/otto +otto/otto-* +tools/tester/testdata/ +tools/tester/tester +tools/gen-jscore/gen-jscore +tools/gen-tokens/gen-tokens .idea dist/ .vscode/ diff --git a/test/tester.go b/test/tester.go deleted file mode 100644 index 4018dbf..0000000 --- a/test/tester.go +++ /dev/null @@ -1,227 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "io" - "net/http" - "os" - "regexp" - "strings" - "sync" - "text/tabwriter" - "time" - - "github.com/robertkrimen/otto" - "github.com/robertkrimen/otto/parser" -) - -var ( - flagTest *bool = flag.Bool("test", false, "") - flagTeport *bool = flag.Bool("report", false, "") -) - -var ( - matchReferenceErrorNotDefined = regexp.MustCompile(`^ReferenceError: \S+ is not defined$`) - matchLookahead = regexp.MustCompile(`Invalid regular expression: re2: Invalid \(\?[=!]\) `) - matchBackreference = regexp.MustCompile(`Invalid regular expression: re2: Invalid \\\d `) - matchTypeErrorUndefined = regexp.MustCompile(`^TypeError: Cannot access member '[^']+' of undefined$`) -) - -var target = map[string]string{ - "test-angular-bindonce.js": "fail", // (anonymous): Line 1:944 Unexpected token ( (and 40 more errors) - "test-jsforce.js": "fail", // (anonymous): Line 9:28329 RuneError (and 5 more errors) - "test-chaplin.js": "parse", // Error: Chaplin requires Common.js or AMD modules - "test-dropbox.js.js": "parse", // Error: dropbox.js loaded in an unsupported JavaScript environment. - "test-epitome.js": "parse", // TypeError: undefined is not a function - "test-portal.js": "parse", // TypeError - "test-reactive-coffee.js": "parse", // Dependencies are not met for reactive: _ and $ not found - "test-scriptaculous.js": "parse", // script.aculo.us requires the Prototype JavaScript framework >= 1.6.0.3 - "test-waypoints.js": "parse", // TypeError: undefined is not a function - "test-webuploader.js": "parse", // Error: `jQuery` is undefined - "test-xuijs.js": "parse", // TypeError: undefined is not a function -} - -// http://cdnjs.com/ -// http://api.cdnjs.com/libraries - -type libraries struct { - Results []library `json:"results"` -} - -type library struct { - Name string `json:"name"` - Latest string `json:"latest"` -} - -func (l library) fetch() error { - if !strings.HasSuffix(l.Latest, ".js") { - return nil - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, l.Latest, nil) - if err != nil { - return fmt.Errorf("request library %q: %w", l.Name, err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("get library %q: %w", l.Name, err) - } - defer resp.Body.Close() //nolint: errcheck - - f, err := os.Create("test-" + l.Name + ".js") - if err != nil { - return fmt.Errorf("create library %q: %w", l.Name, err) - } - - if _, err = io.Copy(f, resp.Body); err != nil { - return fmt.Errorf("write library %q: %w", l.Name, err) - } - - return nil -} - -func test(filename string) error { - script, err := os.ReadFile(filename) //nolint: gosec - if err != nil { - return err - } - - if !*flagTeport { - fmt.Fprintln(os.Stdout, filename, len(script)) - } - - parse := false - if target[filename] != "parse" { - vm := otto.New() - if _, err = vm.Run(string(script)); err != nil { - value := err.Error() - switch { - case matchReferenceErrorNotDefined.MatchString(value), - matchTypeErrorUndefined.MatchString(value), - matchLookahead.MatchString(value), - matchBackreference.MatchString(value): - default: - return err - } - parse = true - } - } - - if parse { - _, err = parser.ParseFile(nil, filename, string(script), parser.IgnoreRegExpErrors) - if err != nil { - return err - } - target[filename] = "parse" - } - - return nil -} - -func fetchAll() error { - resp, err := http.Get("http://api.cdnjs.com/libraries") //nolint: noctx - if err != nil { - return err - } - defer resp.Body.Close() //nolint: errcheck - - var libs libraries - dec := json.NewDecoder(resp.Body) - if err := dec.Decode(&libs); err != nil { - return fmt.Errorf("json decode: %w", err) - } - - var wg sync.WaitGroup - errs := make(chan error, 5) - for _, lib := range libs.Results { - wg.Add(1) - go func(lib library) { - defer wg.Done() - errs <- lib.fetch() - }(lib) - } - - defer func() { - wg.Wait() - close(errs) - }() - - for err := range errs { - if err != nil { - return err - } - } - - return nil -} - -func report() error { - files, err := os.ReadDir(".") - if err != nil { - return fmt.Errorf("read dir: %w", err) - } - - writer := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0) - fmt.Fprintln(writer, "", "\t| Status") - fmt.Fprintln(writer, "---", "\t| ---") - for _, file := range files { - filename := file.Name() - if !strings.HasPrefix(filename, "test-") { - continue - } - err := test(filename) - option := target[filename] - name := strings.TrimPrefix(strings.TrimSuffix(filename, ".js"), "test-") - if err != nil { - fmt.Fprintln(writer, name, "\t| fail") - continue - } - - switch option { - case "": - fmt.Fprintln(writer, name, "\t| pass") - case "parse": - fmt.Fprintln(writer, name, "\t| pass (parse)") - case "re2": - fmt.Fprintln(writer, name, "\t| unknown (re2)") - } - } - return writer.Flush() -} - -func main() { - flag.Parse() - - var filename string - err := func() error { - if flag.Arg(0) == "fetch" { - return fetchAll() - } - - if *flagTeport { - return report() - } - - filename = flag.Arg(0) - return test(filename) - }() - if err != nil { - if filename != "" { - if *flagTest && target[filename] == "fail" { - goto exit - } - fmt.Fprintf(os.Stderr, "%s: %s\n", filename, err.Error()) - } else { - fmt.Fprintln(os.Stderr, err) - } - os.Exit(64) - } -exit: -} diff --git a/tools/tester/main.go b/tools/tester/main.go new file mode 100644 index 0000000..5641e6f --- /dev/null +++ b/tools/tester/main.go @@ -0,0 +1,341 @@ +// Command tester automates the ability to download a suite of JavaScript libraries from a CDN and check if otto can handle them. +// +// It provides two commands via flags: +// * -fetch = Fetch all libraries from the CDN and store them in local testdata directory. +// * -report [file1 file2 ... fileN] = Report the results of trying to run the given or if empty all libraries in the testdata directory. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "sync" + "text/tabwriter" + "time" + + "github.com/robertkrimen/otto" + "github.com/robertkrimen/otto/parser" +) + +const ( + // dataDir is where the libraries are downloaded to. + dataDir = "testdata" + + // downloadWorkers is the number of workers that process downloads. + downloadWorkers = 40 + + // librariesURL is the source for JavaScript libraries for testing. + librariesURL = "http://api.cdnjs.com/libraries" + + // requestTimeout is the maximum time we wait for a request to complete. + requestTimeout = time.Second * 20 +) + +var ( + // testWorkers is the number of workers that process report. + testWorkers = min(10, runtime.GOMAXPROCS(0)) + + // noopConsole is a noopConsole which ignore log requests. + noopConsole = map[string]interface{}{ + "log": func(call otto.FunctionCall) otto.Value { + return otto.UndefinedValue() + }, + } +) + +var ( + matchReferenceErrorNotDefined = regexp.MustCompile(`^ReferenceError: \S+ is not defined$`) + matchLookahead = regexp.MustCompile(`Invalid regular expression: re2: Invalid \(\?[=!]\) `) + matchBackReference = regexp.MustCompile(`Invalid regular expression: re2: Invalid \\\d `) + matchTypeErrorUndefined = regexp.MustCompile(`^TypeError: Cannot access member '[^']+' of undefined$`) +) + +// broken identifies libraries which fail with a fatal error, so must be skipped. +var broken = map[string]string{ + "lets-plot.js": "stack overflow", + "knockout-es5.js": "stack overflow", + "sendbird-calls.js": "runtime: out of memory", +} + +func min(a, b int) int { + if a > b { + return b + } + return a +} + +// libraries represents fetch all libraries response. +type libraries struct { + Results []library `json:"results"` +} + +// library represents a single library in a libraries response. +type library struct { + Name string `json:"name"` + Latest string `json:"latest"` +} + +// fetch fetches itself and stores it in the dataDir. +func (l library) fetch() error { + if !strings.HasSuffix(l.Latest, ".js") { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, l.Latest, nil) + if err != nil { + return fmt.Errorf("request library %q: %w", l.Name, err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("get library %q: %w", l.Name, err) + } + defer resp.Body.Close() //nolint: errcheck + + name := l.Name + if !strings.HasSuffix(name, ".js") { + name += ".js" + } + + f, err := os.Create(filepath.Join(dataDir, name)) //nolint: gosec + if err != nil { + return fmt.Errorf("create library %q: %w", l.Name, err) + } + + if _, err = io.Copy(f, resp.Body); err != nil { + return fmt.Errorf("write library %q: %w", l.Name, err) + } + + return nil +} + +// test runs the code from filename returning the time it took and any error +// encountered when running a full parse without IgnoreRegExpErrors in parseError. +func test(filename string) (took time.Duration, parseError, err error) { //nolint: nonamedreturns + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic on %q: %v", filename, r) + } + }() + now := time.Now() + defer func() { + // Always set took. + took = time.Since(now) + }() + + if val := broken[filepath.Base(filename)]; val != "" { + return 0, nil, fmt.Errorf("fatal %q", val) + } + + script, err := os.ReadFile(filename) //nolint: gosec + if err != nil { + return 0, nil, err + } + + vm := otto.New() + if err := vm.Set("console", noopConsole); err != nil { + return 0, nil, fmt.Errorf("set console: %w", err) + } + + prog, err := parser.ParseFile(nil, filename, string(script), 0) + if err != nil { + val := err.Error() + switch { + case matchReferenceErrorNotDefined.MatchString(val), + matchTypeErrorUndefined.MatchString(val), + matchLookahead.MatchString(val), + matchBackReference.MatchString(val): + // RegExp issues, retry with IgnoreRegExpErrors. + parseError = err + if _, err = parser.ParseFile(nil, filename, string(script), parser.IgnoreRegExpErrors); err != nil { + return 0, nil, err + } + return 0, parseError, nil + default: + return 0, nil, err + } + } + + _, err = vm.Run(prog) + return 0, nil, err +} + +// fetchAll fetches all files from src. +func fetchAll(src string) error { + ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, librariesURL, nil) + if err != nil { + return fmt.Errorf("request libraries %q: %w", src, err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("get libraries %q: %w", src, err) + } + defer resp.Body.Close() //nolint: errcheck + + var libs libraries + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&libs); err != nil { + return fmt.Errorf("json decode: %w", err) + } + + if err := os.Mkdir(dataDir, 0o750); err != nil && !errors.Is(err, fs.ErrExist) { + return fmt.Errorf("mkdir: %w", err) + } + + var wg sync.WaitGroup + work := make(chan library, downloadWorkers) + errs := make(chan error, len(libs.Results)) + wg.Add(downloadWorkers) + for i := 0; i < downloadWorkers; i++ { + go func() { + defer wg.Done() + for lib := range work { + fmt.Fprint(os.Stdout, ".") + errs <- lib.fetch() + } + }() + } + + fmt.Fprintf(os.Stdout, "Downloading %d libraries with %d workers ...", len(libs.Results), downloadWorkers) + wg.Add(1) + go func() { + defer wg.Done() + for _, lib := range libs.Results { + work <- lib + } + close(work) + }() + + wg.Wait() + close(errs) + fmt.Fprintln(os.Stdout, " done") + + for e := range errs { + if e != nil { + fmt.Fprintln(os.Stderr, e) + err = e + } + } + + return err +} + +// result represents the result from a test. +type result struct { + filename string + err error + parseError error + took time.Duration +} + +// report runs test for all specified files, if none a specified all +// JavaScript files in our dataDir, outputting the results. +func report(files []string) error { + if len(files) == 0 { + var err error + files, err = filepath.Glob(filepath.Join(dataDir, "*.js")) + if err != nil { + return fmt.Errorf("read dir: %w", err) + } + } + + var wg sync.WaitGroup + workers := min(testWorkers, len(files)) + work := make(chan string, workers) + results := make(chan result, len(files)) + wg.Add(workers) + for i := 0; i < workers; i++ { + go func() { + defer wg.Done() + for f := range work { + fmt.Fprint(os.Stdout, ".") + took, parseError, err := test(f) + results <- result{ + filename: f, + err: err, + parseError: parseError, + took: took, + } + } + }() + } + + fmt.Fprintf(os.Stdout, "Reporting on %d libs with %d workers...", len(files), workers) + wg.Add(1) + go func() { + defer wg.Done() + for _, f := range files { + work <- f + } + close(work) + }() + + wg.Wait() + close(results) + fmt.Fprintln(os.Stdout, " done") + + var fail, pass, parse int + writer := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0) + fmt.Fprintln(writer, "Library", "\t| Took", "\t| Status") + fmt.Fprintln(writer, "-------", "\t| ----", "\t| ------") + + for res := range results { + switch { + case res.err != nil: + fmt.Fprintf(writer, "%s\t| %v\t| fail: %v\n", res.filename, res.took, res.err) + fail++ + case res.parseError != nil: + fmt.Fprintf(writer, "%s\t| %v\t| pass parse: %v\n", res.filename, res.took, res.parseError) + parse++ + default: + fmt.Fprintf(writer, "%s\t| %v\t| pass\n", res.filename, res.took) + pass++ + } + } + + if err := writer.Flush(); err != nil { + return fmt.Errorf("flush: %w", err) + } + + fmt.Fprintf(os.Stdout, "\nSummary:\n - %d passes\n - %d parse passes\n - %d fails\n", pass, parse, fail) + + return nil +} + +func main() { + flagFetch := flag.Bool("fetch", false, "fetch all libraries for testing") + flagReport := flag.Bool("report", false, "test and report the named files or all libraries if non specified") + flag.Parse() + + var err error + switch { + case *flagFetch: + err = fetchAll(librariesURL) + case *flagReport: + err = report(flag.Args()) + default: + flag.PrintDefaults() + err = fmt.Errorf("missing flag") + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(64) + } +}