1
0
mirror of https://github.com/robertkrimen/otto synced 2025-10-05 19:19:10 +08:00

fix: tester (#478)

Fix the way tester works, move it into the tools directory and clean up
how it processes files including adding more data to the summary about
what failed and why.
This commit is contained in:
Steven Hartland 2022-12-05 23:15:21 +00:00 committed by GitHub
parent 898a889ed4
commit 9230e2b888
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 348 additions and 232 deletions

12
.gitignore vendored
View File

@ -1,8 +1,10 @@
/.test
/otto/otto
/otto/otto-*
/test/test-*.js
/test/tester
.test
otto/otto
otto/otto-*
tools/tester/testdata/
tools/tester/tester
tools/gen-jscore/gen-jscore
tools/gen-tokens/gen-tokens
.idea
dist/
.vscode/

View File

@ -1,227 +0,0 @@
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"io"
"net/http"
"os"
"regexp"
"strings"
"sync"
"text/tabwriter"
"time"
"github.com/robertkrimen/otto"
"github.com/robertkrimen/otto/parser"
)
var (
flagTest *bool = flag.Bool("test", false, "")
flagTeport *bool = flag.Bool("report", false, "")
)
var (
matchReferenceErrorNotDefined = regexp.MustCompile(`^ReferenceError: \S+ is not defined$`)
matchLookahead = regexp.MustCompile(`Invalid regular expression: re2: Invalid \(\?[=!]\) <lookahead>`)
matchBackreference = regexp.MustCompile(`Invalid regular expression: re2: Invalid \\\d <backreference>`)
matchTypeErrorUndefined = regexp.MustCompile(`^TypeError: Cannot access member '[^']+' of undefined$`)
)
var target = map[string]string{
"test-angular-bindonce.js": "fail", // (anonymous): Line 1:944 Unexpected token ( (and 40 more errors)
"test-jsforce.js": "fail", // (anonymous): Line 9:28329 RuneError (and 5 more errors)
"test-chaplin.js": "parse", // Error: Chaplin requires Common.js or AMD modules
"test-dropbox.js.js": "parse", // Error: dropbox.js loaded in an unsupported JavaScript environment.
"test-epitome.js": "parse", // TypeError: undefined is not a function
"test-portal.js": "parse", // TypeError
"test-reactive-coffee.js": "parse", // Dependencies are not met for reactive: _ and $ not found
"test-scriptaculous.js": "parse", // script.aculo.us requires the Prototype JavaScript framework >= 1.6.0.3
"test-waypoints.js": "parse", // TypeError: undefined is not a function
"test-webuploader.js": "parse", // Error: `jQuery` is undefined
"test-xuijs.js": "parse", // TypeError: undefined is not a function
}
// http://cdnjs.com/
// http://api.cdnjs.com/libraries
type libraries struct {
Results []library `json:"results"`
}
type library struct {
Name string `json:"name"`
Latest string `json:"latest"`
}
func (l library) fetch() error {
if !strings.HasSuffix(l.Latest, ".js") {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*20)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, l.Latest, nil)
if err != nil {
return fmt.Errorf("request library %q: %w", l.Name, err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("get library %q: %w", l.Name, err)
}
defer resp.Body.Close() //nolint: errcheck
f, err := os.Create("test-" + l.Name + ".js")
if err != nil {
return fmt.Errorf("create library %q: %w", l.Name, err)
}
if _, err = io.Copy(f, resp.Body); err != nil {
return fmt.Errorf("write library %q: %w", l.Name, err)
}
return nil
}
func test(filename string) error {
script, err := os.ReadFile(filename) //nolint: gosec
if err != nil {
return err
}
if !*flagTeport {
fmt.Fprintln(os.Stdout, filename, len(script))
}
parse := false
if target[filename] != "parse" {
vm := otto.New()
if _, err = vm.Run(string(script)); err != nil {
value := err.Error()
switch {
case matchReferenceErrorNotDefined.MatchString(value),
matchTypeErrorUndefined.MatchString(value),
matchLookahead.MatchString(value),
matchBackreference.MatchString(value):
default:
return err
}
parse = true
}
}
if parse {
_, err = parser.ParseFile(nil, filename, string(script), parser.IgnoreRegExpErrors)
if err != nil {
return err
}
target[filename] = "parse"
}
return nil
}
func fetchAll() error {
resp, err := http.Get("http://api.cdnjs.com/libraries") //nolint: noctx
if err != nil {
return err
}
defer resp.Body.Close() //nolint: errcheck
var libs libraries
dec := json.NewDecoder(resp.Body)
if err := dec.Decode(&libs); err != nil {
return fmt.Errorf("json decode: %w", err)
}
var wg sync.WaitGroup
errs := make(chan error, 5)
for _, lib := range libs.Results {
wg.Add(1)
go func(lib library) {
defer wg.Done()
errs <- lib.fetch()
}(lib)
}
defer func() {
wg.Wait()
close(errs)
}()
for err := range errs {
if err != nil {
return err
}
}
return nil
}
func report() error {
files, err := os.ReadDir(".")
if err != nil {
return fmt.Errorf("read dir: %w", err)
}
writer := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)
fmt.Fprintln(writer, "", "\t| Status")
fmt.Fprintln(writer, "---", "\t| ---")
for _, file := range files {
filename := file.Name()
if !strings.HasPrefix(filename, "test-") {
continue
}
err := test(filename)
option := target[filename]
name := strings.TrimPrefix(strings.TrimSuffix(filename, ".js"), "test-")
if err != nil {
fmt.Fprintln(writer, name, "\t| fail")
continue
}
switch option {
case "":
fmt.Fprintln(writer, name, "\t| pass")
case "parse":
fmt.Fprintln(writer, name, "\t| pass (parse)")
case "re2":
fmt.Fprintln(writer, name, "\t| unknown (re2)")
}
}
return writer.Flush()
}
func main() {
flag.Parse()
var filename string
err := func() error {
if flag.Arg(0) == "fetch" {
return fetchAll()
}
if *flagTeport {
return report()
}
filename = flag.Arg(0)
return test(filename)
}()
if err != nil {
if filename != "" {
if *flagTest && target[filename] == "fail" {
goto exit
}
fmt.Fprintf(os.Stderr, "%s: %s\n", filename, err.Error())
} else {
fmt.Fprintln(os.Stderr, err)
}
os.Exit(64)
}
exit:
}

341
tools/tester/main.go Normal file
View File

@ -0,0 +1,341 @@
// Command tester automates the ability to download a suite of JavaScript libraries from a CDN and check if otto can handle them.
//
// It provides two commands via flags:
// * -fetch = Fetch all libraries from the CDN and store them in local testdata directory.
// * -report [file1 file2 ... fileN] = Report the results of trying to run the given or if empty all libraries in the testdata directory.
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"io/fs"
"net/http"
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
"sync"
"text/tabwriter"
"time"
"github.com/robertkrimen/otto"
"github.com/robertkrimen/otto/parser"
)
const (
// dataDir is where the libraries are downloaded to.
dataDir = "testdata"
// downloadWorkers is the number of workers that process downloads.
downloadWorkers = 40
// librariesURL is the source for JavaScript libraries for testing.
librariesURL = "http://api.cdnjs.com/libraries"
// requestTimeout is the maximum time we wait for a request to complete.
requestTimeout = time.Second * 20
)
var (
// testWorkers is the number of workers that process report.
testWorkers = min(10, runtime.GOMAXPROCS(0))
// noopConsole is a noopConsole which ignore log requests.
noopConsole = map[string]interface{}{
"log": func(call otto.FunctionCall) otto.Value {
return otto.UndefinedValue()
},
}
)
var (
matchReferenceErrorNotDefined = regexp.MustCompile(`^ReferenceError: \S+ is not defined$`)
matchLookahead = regexp.MustCompile(`Invalid regular expression: re2: Invalid \(\?[=!]\) <lookahead>`)
matchBackReference = regexp.MustCompile(`Invalid regular expression: re2: Invalid \\\d <backreference>`)
matchTypeErrorUndefined = regexp.MustCompile(`^TypeError: Cannot access member '[^']+' of undefined$`)
)
// broken identifies libraries which fail with a fatal error, so must be skipped.
var broken = map[string]string{
"lets-plot.js": "stack overflow",
"knockout-es5.js": "stack overflow",
"sendbird-calls.js": "runtime: out of memory",
}
func min(a, b int) int {
if a > b {
return b
}
return a
}
// libraries represents fetch all libraries response.
type libraries struct {
Results []library `json:"results"`
}
// library represents a single library in a libraries response.
type library struct {
Name string `json:"name"`
Latest string `json:"latest"`
}
// fetch fetches itself and stores it in the dataDir.
func (l library) fetch() error {
if !strings.HasSuffix(l.Latest, ".js") {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, l.Latest, nil)
if err != nil {
return fmt.Errorf("request library %q: %w", l.Name, err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("get library %q: %w", l.Name, err)
}
defer resp.Body.Close() //nolint: errcheck
name := l.Name
if !strings.HasSuffix(name, ".js") {
name += ".js"
}
f, err := os.Create(filepath.Join(dataDir, name)) //nolint: gosec
if err != nil {
return fmt.Errorf("create library %q: %w", l.Name, err)
}
if _, err = io.Copy(f, resp.Body); err != nil {
return fmt.Errorf("write library %q: %w", l.Name, err)
}
return nil
}
// test runs the code from filename returning the time it took and any error
// encountered when running a full parse without IgnoreRegExpErrors in parseError.
func test(filename string) (took time.Duration, parseError, err error) { //nolint: nonamedreturns
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("panic on %q: %v", filename, r)
}
}()
now := time.Now()
defer func() {
// Always set took.
took = time.Since(now)
}()
if val := broken[filepath.Base(filename)]; val != "" {
return 0, nil, fmt.Errorf("fatal %q", val)
}
script, err := os.ReadFile(filename) //nolint: gosec
if err != nil {
return 0, nil, err
}
vm := otto.New()
if err := vm.Set("console", noopConsole); err != nil {
return 0, nil, fmt.Errorf("set console: %w", err)
}
prog, err := parser.ParseFile(nil, filename, string(script), 0)
if err != nil {
val := err.Error()
switch {
case matchReferenceErrorNotDefined.MatchString(val),
matchTypeErrorUndefined.MatchString(val),
matchLookahead.MatchString(val),
matchBackReference.MatchString(val):
// RegExp issues, retry with IgnoreRegExpErrors.
parseError = err
if _, err = parser.ParseFile(nil, filename, string(script), parser.IgnoreRegExpErrors); err != nil {
return 0, nil, err
}
return 0, parseError, nil
default:
return 0, nil, err
}
}
_, err = vm.Run(prog)
return 0, nil, err
}
// fetchAll fetches all files from src.
func fetchAll(src string) error {
ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, librariesURL, nil)
if err != nil {
return fmt.Errorf("request libraries %q: %w", src, err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("get libraries %q: %w", src, err)
}
defer resp.Body.Close() //nolint: errcheck
var libs libraries
dec := json.NewDecoder(resp.Body)
if err := dec.Decode(&libs); err != nil {
return fmt.Errorf("json decode: %w", err)
}
if err := os.Mkdir(dataDir, 0o750); err != nil && !errors.Is(err, fs.ErrExist) {
return fmt.Errorf("mkdir: %w", err)
}
var wg sync.WaitGroup
work := make(chan library, downloadWorkers)
errs := make(chan error, len(libs.Results))
wg.Add(downloadWorkers)
for i := 0; i < downloadWorkers; i++ {
go func() {
defer wg.Done()
for lib := range work {
fmt.Fprint(os.Stdout, ".")
errs <- lib.fetch()
}
}()
}
fmt.Fprintf(os.Stdout, "Downloading %d libraries with %d workers ...", len(libs.Results), downloadWorkers)
wg.Add(1)
go func() {
defer wg.Done()
for _, lib := range libs.Results {
work <- lib
}
close(work)
}()
wg.Wait()
close(errs)
fmt.Fprintln(os.Stdout, " done")
for e := range errs {
if e != nil {
fmt.Fprintln(os.Stderr, e)
err = e
}
}
return err
}
// result represents the result from a test.
type result struct {
filename string
err error
parseError error
took time.Duration
}
// report runs test for all specified files, if none a specified all
// JavaScript files in our dataDir, outputting the results.
func report(files []string) error {
if len(files) == 0 {
var err error
files, err = filepath.Glob(filepath.Join(dataDir, "*.js"))
if err != nil {
return fmt.Errorf("read dir: %w", err)
}
}
var wg sync.WaitGroup
workers := min(testWorkers, len(files))
work := make(chan string, workers)
results := make(chan result, len(files))
wg.Add(workers)
for i := 0; i < workers; i++ {
go func() {
defer wg.Done()
for f := range work {
fmt.Fprint(os.Stdout, ".")
took, parseError, err := test(f)
results <- result{
filename: f,
err: err,
parseError: parseError,
took: took,
}
}
}()
}
fmt.Fprintf(os.Stdout, "Reporting on %d libs with %d workers...", len(files), workers)
wg.Add(1)
go func() {
defer wg.Done()
for _, f := range files {
work <- f
}
close(work)
}()
wg.Wait()
close(results)
fmt.Fprintln(os.Stdout, " done")
var fail, pass, parse int
writer := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)
fmt.Fprintln(writer, "Library", "\t| Took", "\t| Status")
fmt.Fprintln(writer, "-------", "\t| ----", "\t| ------")
for res := range results {
switch {
case res.err != nil:
fmt.Fprintf(writer, "%s\t| %v\t| fail: %v\n", res.filename, res.took, res.err)
fail++
case res.parseError != nil:
fmt.Fprintf(writer, "%s\t| %v\t| pass parse: %v\n", res.filename, res.took, res.parseError)
parse++
default:
fmt.Fprintf(writer, "%s\t| %v\t| pass\n", res.filename, res.took)
pass++
}
}
if err := writer.Flush(); err != nil {
return fmt.Errorf("flush: %w", err)
}
fmt.Fprintf(os.Stdout, "\nSummary:\n - %d passes\n - %d parse passes\n - %d fails\n", pass, parse, fail)
return nil
}
func main() {
flagFetch := flag.Bool("fetch", false, "fetch all libraries for testing")
flagReport := flag.Bool("report", false, "test and report the named files or all libraries if non specified")
flag.Parse()
var err error
switch {
case *flagFetch:
err = fetchAll(librariesURL)
case *flagReport:
err = report(flag.Args())
default:
flag.PrintDefaults()
err = fmt.Errorf("missing flag")
}
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(64)
}
}