mirror of
https://github.com/robertkrimen/otto
synced 2025-10-05 19:19:10 +08:00
fix: tester (#478)
Fix the way tester works, move it into the tools directory and clean up how it processes files including adding more data to the summary about what failed and why.
This commit is contained in:
parent
898a889ed4
commit
9230e2b888
12
.gitignore
vendored
12
.gitignore
vendored
|
@ -1,8 +1,10 @@
|
|||
/.test
|
||||
/otto/otto
|
||||
/otto/otto-*
|
||||
/test/test-*.js
|
||||
/test/tester
|
||||
.test
|
||||
otto/otto
|
||||
otto/otto-*
|
||||
tools/tester/testdata/
|
||||
tools/tester/tester
|
||||
tools/gen-jscore/gen-jscore
|
||||
tools/gen-tokens/gen-tokens
|
||||
.idea
|
||||
dist/
|
||||
.vscode/
|
||||
|
|
227
test/tester.go
227
test/tester.go
|
@ -1,227 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/robertkrimen/otto"
|
||||
"github.com/robertkrimen/otto/parser"
|
||||
)
|
||||
|
||||
var (
|
||||
flagTest *bool = flag.Bool("test", false, "")
|
||||
flagTeport *bool = flag.Bool("report", false, "")
|
||||
)
|
||||
|
||||
var (
|
||||
matchReferenceErrorNotDefined = regexp.MustCompile(`^ReferenceError: \S+ is not defined$`)
|
||||
matchLookahead = regexp.MustCompile(`Invalid regular expression: re2: Invalid \(\?[=!]\) <lookahead>`)
|
||||
matchBackreference = regexp.MustCompile(`Invalid regular expression: re2: Invalid \\\d <backreference>`)
|
||||
matchTypeErrorUndefined = regexp.MustCompile(`^TypeError: Cannot access member '[^']+' of undefined$`)
|
||||
)
|
||||
|
||||
var target = map[string]string{
|
||||
"test-angular-bindonce.js": "fail", // (anonymous): Line 1:944 Unexpected token ( (and 40 more errors)
|
||||
"test-jsforce.js": "fail", // (anonymous): Line 9:28329 RuneError (and 5 more errors)
|
||||
"test-chaplin.js": "parse", // Error: Chaplin requires Common.js or AMD modules
|
||||
"test-dropbox.js.js": "parse", // Error: dropbox.js loaded in an unsupported JavaScript environment.
|
||||
"test-epitome.js": "parse", // TypeError: undefined is not a function
|
||||
"test-portal.js": "parse", // TypeError
|
||||
"test-reactive-coffee.js": "parse", // Dependencies are not met for reactive: _ and $ not found
|
||||
"test-scriptaculous.js": "parse", // script.aculo.us requires the Prototype JavaScript framework >= 1.6.0.3
|
||||
"test-waypoints.js": "parse", // TypeError: undefined is not a function
|
||||
"test-webuploader.js": "parse", // Error: `jQuery` is undefined
|
||||
"test-xuijs.js": "parse", // TypeError: undefined is not a function
|
||||
}
|
||||
|
||||
// http://cdnjs.com/
|
||||
// http://api.cdnjs.com/libraries
|
||||
|
||||
type libraries struct {
|
||||
Results []library `json:"results"`
|
||||
}
|
||||
|
||||
type library struct {
|
||||
Name string `json:"name"`
|
||||
Latest string `json:"latest"`
|
||||
}
|
||||
|
||||
func (l library) fetch() error {
|
||||
if !strings.HasSuffix(l.Latest, ".js") {
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*20)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, l.Latest, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("request library %q: %w", l.Name, err)
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get library %q: %w", l.Name, err)
|
||||
}
|
||||
defer resp.Body.Close() //nolint: errcheck
|
||||
|
||||
f, err := os.Create("test-" + l.Name + ".js")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create library %q: %w", l.Name, err)
|
||||
}
|
||||
|
||||
if _, err = io.Copy(f, resp.Body); err != nil {
|
||||
return fmt.Errorf("write library %q: %w", l.Name, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func test(filename string) error {
|
||||
script, err := os.ReadFile(filename) //nolint: gosec
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !*flagTeport {
|
||||
fmt.Fprintln(os.Stdout, filename, len(script))
|
||||
}
|
||||
|
||||
parse := false
|
||||
if target[filename] != "parse" {
|
||||
vm := otto.New()
|
||||
if _, err = vm.Run(string(script)); err != nil {
|
||||
value := err.Error()
|
||||
switch {
|
||||
case matchReferenceErrorNotDefined.MatchString(value),
|
||||
matchTypeErrorUndefined.MatchString(value),
|
||||
matchLookahead.MatchString(value),
|
||||
matchBackreference.MatchString(value):
|
||||
default:
|
||||
return err
|
||||
}
|
||||
parse = true
|
||||
}
|
||||
}
|
||||
|
||||
if parse {
|
||||
_, err = parser.ParseFile(nil, filename, string(script), parser.IgnoreRegExpErrors)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target[filename] = "parse"
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchAll() error {
|
||||
resp, err := http.Get("http://api.cdnjs.com/libraries") //nolint: noctx
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close() //nolint: errcheck
|
||||
|
||||
var libs libraries
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
if err := dec.Decode(&libs); err != nil {
|
||||
return fmt.Errorf("json decode: %w", err)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 5)
|
||||
for _, lib := range libs.Results {
|
||||
wg.Add(1)
|
||||
go func(lib library) {
|
||||
defer wg.Done()
|
||||
errs <- lib.fetch()
|
||||
}(lib)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
}()
|
||||
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func report() error {
|
||||
files, err := os.ReadDir(".")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read dir: %w", err)
|
||||
}
|
||||
|
||||
writer := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)
|
||||
fmt.Fprintln(writer, "", "\t| Status")
|
||||
fmt.Fprintln(writer, "---", "\t| ---")
|
||||
for _, file := range files {
|
||||
filename := file.Name()
|
||||
if !strings.HasPrefix(filename, "test-") {
|
||||
continue
|
||||
}
|
||||
err := test(filename)
|
||||
option := target[filename]
|
||||
name := strings.TrimPrefix(strings.TrimSuffix(filename, ".js"), "test-")
|
||||
if err != nil {
|
||||
fmt.Fprintln(writer, name, "\t| fail")
|
||||
continue
|
||||
}
|
||||
|
||||
switch option {
|
||||
case "":
|
||||
fmt.Fprintln(writer, name, "\t| pass")
|
||||
case "parse":
|
||||
fmt.Fprintln(writer, name, "\t| pass (parse)")
|
||||
case "re2":
|
||||
fmt.Fprintln(writer, name, "\t| unknown (re2)")
|
||||
}
|
||||
}
|
||||
return writer.Flush()
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
var filename string
|
||||
err := func() error {
|
||||
if flag.Arg(0) == "fetch" {
|
||||
return fetchAll()
|
||||
}
|
||||
|
||||
if *flagTeport {
|
||||
return report()
|
||||
}
|
||||
|
||||
filename = flag.Arg(0)
|
||||
return test(filename)
|
||||
}()
|
||||
if err != nil {
|
||||
if filename != "" {
|
||||
if *flagTest && target[filename] == "fail" {
|
||||
goto exit
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "%s: %s\n", filename, err.Error())
|
||||
} else {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
}
|
||||
os.Exit(64)
|
||||
}
|
||||
exit:
|
||||
}
|
341
tools/tester/main.go
Normal file
341
tools/tester/main.go
Normal file
|
@ -0,0 +1,341 @@
|
|||
// Command tester automates the ability to download a suite of JavaScript libraries from a CDN and check if otto can handle them.
|
||||
//
|
||||
// It provides two commands via flags:
|
||||
// * -fetch = Fetch all libraries from the CDN and store them in local testdata directory.
|
||||
// * -report [file1 file2 ... fileN] = Report the results of trying to run the given or if empty all libraries in the testdata directory.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/robertkrimen/otto"
|
||||
"github.com/robertkrimen/otto/parser"
|
||||
)
|
||||
|
||||
const (
|
||||
// dataDir is where the libraries are downloaded to.
|
||||
dataDir = "testdata"
|
||||
|
||||
// downloadWorkers is the number of workers that process downloads.
|
||||
downloadWorkers = 40
|
||||
|
||||
// librariesURL is the source for JavaScript libraries for testing.
|
||||
librariesURL = "http://api.cdnjs.com/libraries"
|
||||
|
||||
// requestTimeout is the maximum time we wait for a request to complete.
|
||||
requestTimeout = time.Second * 20
|
||||
)
|
||||
|
||||
var (
|
||||
// testWorkers is the number of workers that process report.
|
||||
testWorkers = min(10, runtime.GOMAXPROCS(0))
|
||||
|
||||
// noopConsole is a noopConsole which ignore log requests.
|
||||
noopConsole = map[string]interface{}{
|
||||
"log": func(call otto.FunctionCall) otto.Value {
|
||||
return otto.UndefinedValue()
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
var (
|
||||
matchReferenceErrorNotDefined = regexp.MustCompile(`^ReferenceError: \S+ is not defined$`)
|
||||
matchLookahead = regexp.MustCompile(`Invalid regular expression: re2: Invalid \(\?[=!]\) <lookahead>`)
|
||||
matchBackReference = regexp.MustCompile(`Invalid regular expression: re2: Invalid \\\d <backreference>`)
|
||||
matchTypeErrorUndefined = regexp.MustCompile(`^TypeError: Cannot access member '[^']+' of undefined$`)
|
||||
)
|
||||
|
||||
// broken identifies libraries which fail with a fatal error, so must be skipped.
|
||||
var broken = map[string]string{
|
||||
"lets-plot.js": "stack overflow",
|
||||
"knockout-es5.js": "stack overflow",
|
||||
"sendbird-calls.js": "runtime: out of memory",
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a > b {
|
||||
return b
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// libraries represents fetch all libraries response.
|
||||
type libraries struct {
|
||||
Results []library `json:"results"`
|
||||
}
|
||||
|
||||
// library represents a single library in a libraries response.
|
||||
type library struct {
|
||||
Name string `json:"name"`
|
||||
Latest string `json:"latest"`
|
||||
}
|
||||
|
||||
// fetch fetches itself and stores it in the dataDir.
|
||||
func (l library) fetch() error {
|
||||
if !strings.HasSuffix(l.Latest, ".js") {
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, l.Latest, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("request library %q: %w", l.Name, err)
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get library %q: %w", l.Name, err)
|
||||
}
|
||||
defer resp.Body.Close() //nolint: errcheck
|
||||
|
||||
name := l.Name
|
||||
if !strings.HasSuffix(name, ".js") {
|
||||
name += ".js"
|
||||
}
|
||||
|
||||
f, err := os.Create(filepath.Join(dataDir, name)) //nolint: gosec
|
||||
if err != nil {
|
||||
return fmt.Errorf("create library %q: %w", l.Name, err)
|
||||
}
|
||||
|
||||
if _, err = io.Copy(f, resp.Body); err != nil {
|
||||
return fmt.Errorf("write library %q: %w", l.Name, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// test runs the code from filename returning the time it took and any error
|
||||
// encountered when running a full parse without IgnoreRegExpErrors in parseError.
|
||||
func test(filename string) (took time.Duration, parseError, err error) { //nolint: nonamedreturns
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
err = fmt.Errorf("panic on %q: %v", filename, r)
|
||||
}
|
||||
}()
|
||||
now := time.Now()
|
||||
defer func() {
|
||||
// Always set took.
|
||||
took = time.Since(now)
|
||||
}()
|
||||
|
||||
if val := broken[filepath.Base(filename)]; val != "" {
|
||||
return 0, nil, fmt.Errorf("fatal %q", val)
|
||||
}
|
||||
|
||||
script, err := os.ReadFile(filename) //nolint: gosec
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
|
||||
vm := otto.New()
|
||||
if err := vm.Set("console", noopConsole); err != nil {
|
||||
return 0, nil, fmt.Errorf("set console: %w", err)
|
||||
}
|
||||
|
||||
prog, err := parser.ParseFile(nil, filename, string(script), 0)
|
||||
if err != nil {
|
||||
val := err.Error()
|
||||
switch {
|
||||
case matchReferenceErrorNotDefined.MatchString(val),
|
||||
matchTypeErrorUndefined.MatchString(val),
|
||||
matchLookahead.MatchString(val),
|
||||
matchBackReference.MatchString(val):
|
||||
// RegExp issues, retry with IgnoreRegExpErrors.
|
||||
parseError = err
|
||||
if _, err = parser.ParseFile(nil, filename, string(script), parser.IgnoreRegExpErrors); err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
return 0, parseError, nil
|
||||
default:
|
||||
return 0, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
_, err = vm.Run(prog)
|
||||
return 0, nil, err
|
||||
}
|
||||
|
||||
// fetchAll fetches all files from src.
|
||||
func fetchAll(src string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, librariesURL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("request libraries %q: %w", src, err)
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get libraries %q: %w", src, err)
|
||||
}
|
||||
defer resp.Body.Close() //nolint: errcheck
|
||||
|
||||
var libs libraries
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
if err := dec.Decode(&libs); err != nil {
|
||||
return fmt.Errorf("json decode: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Mkdir(dataDir, 0o750); err != nil && !errors.Is(err, fs.ErrExist) {
|
||||
return fmt.Errorf("mkdir: %w", err)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
work := make(chan library, downloadWorkers)
|
||||
errs := make(chan error, len(libs.Results))
|
||||
wg.Add(downloadWorkers)
|
||||
for i := 0; i < downloadWorkers; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for lib := range work {
|
||||
fmt.Fprint(os.Stdout, ".")
|
||||
errs <- lib.fetch()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stdout, "Downloading %d libraries with %d workers ...", len(libs.Results), downloadWorkers)
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for _, lib := range libs.Results {
|
||||
work <- lib
|
||||
}
|
||||
close(work)
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
fmt.Fprintln(os.Stdout, " done")
|
||||
|
||||
for e := range errs {
|
||||
if e != nil {
|
||||
fmt.Fprintln(os.Stderr, e)
|
||||
err = e
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// result represents the result from a test.
|
||||
type result struct {
|
||||
filename string
|
||||
err error
|
||||
parseError error
|
||||
took time.Duration
|
||||
}
|
||||
|
||||
// report runs test for all specified files, if none a specified all
|
||||
// JavaScript files in our dataDir, outputting the results.
|
||||
func report(files []string) error {
|
||||
if len(files) == 0 {
|
||||
var err error
|
||||
files, err = filepath.Glob(filepath.Join(dataDir, "*.js"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("read dir: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
workers := min(testWorkers, len(files))
|
||||
work := make(chan string, workers)
|
||||
results := make(chan result, len(files))
|
||||
wg.Add(workers)
|
||||
for i := 0; i < workers; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for f := range work {
|
||||
fmt.Fprint(os.Stdout, ".")
|
||||
took, parseError, err := test(f)
|
||||
results <- result{
|
||||
filename: f,
|
||||
err: err,
|
||||
parseError: parseError,
|
||||
took: took,
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stdout, "Reporting on %d libs with %d workers...", len(files), workers)
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for _, f := range files {
|
||||
work <- f
|
||||
}
|
||||
close(work)
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
close(results)
|
||||
fmt.Fprintln(os.Stdout, " done")
|
||||
|
||||
var fail, pass, parse int
|
||||
writer := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)
|
||||
fmt.Fprintln(writer, "Library", "\t| Took", "\t| Status")
|
||||
fmt.Fprintln(writer, "-------", "\t| ----", "\t| ------")
|
||||
|
||||
for res := range results {
|
||||
switch {
|
||||
case res.err != nil:
|
||||
fmt.Fprintf(writer, "%s\t| %v\t| fail: %v\n", res.filename, res.took, res.err)
|
||||
fail++
|
||||
case res.parseError != nil:
|
||||
fmt.Fprintf(writer, "%s\t| %v\t| pass parse: %v\n", res.filename, res.took, res.parseError)
|
||||
parse++
|
||||
default:
|
||||
fmt.Fprintf(writer, "%s\t| %v\t| pass\n", res.filename, res.took)
|
||||
pass++
|
||||
}
|
||||
}
|
||||
|
||||
if err := writer.Flush(); err != nil {
|
||||
return fmt.Errorf("flush: %w", err)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stdout, "\nSummary:\n - %d passes\n - %d parse passes\n - %d fails\n", pass, parse, fail)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
flagFetch := flag.Bool("fetch", false, "fetch all libraries for testing")
|
||||
flagReport := flag.Bool("report", false, "test and report the named files or all libraries if non specified")
|
||||
flag.Parse()
|
||||
|
||||
var err error
|
||||
switch {
|
||||
case *flagFetch:
|
||||
err = fetchAll(librariesURL)
|
||||
case *flagReport:
|
||||
err = report(flag.Args())
|
||||
default:
|
||||
flag.PrintDefaults()
|
||||
err = fmt.Errorf("missing flag")
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(64)
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user