diff --git a/.gitignore b/.gitignore index 69e9225c9a9..51750dad3bb 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,7 @@ backend/server/api/docs/swagger.json backend/server/api/docs/swagger.yaml backend/server/api/docs/docs.go *.result/ + +# python +*.pyc +__pycache__ diff --git a/.licenserc.yaml b/.licenserc.yaml index b11fd825453..aa9cd51ef23 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -34,6 +34,7 @@ header: - '**/env.example' - '**/*.csv' - '**/*.json' + - '**/*.template.json' - '**/*.sql' - '**/*.svg' - '**/*.png' @@ -52,6 +53,7 @@ header: - '**/.eslintrc' - 'deployment/helm/templates/_helpers.tpl' - '**/.nvmrc' + - '**/poetry.lock' diff --git a/backend/Dockerfile b/backend/Dockerfile index dafbcd9918c..aab56730173 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -138,7 +138,9 @@ COPY python/requirements.txt /app/requirements.txt RUN python3 -m pip install --no-cache --upgrade pip setuptools && \ python3 -m pip install --no-cache dbt-mysql dbt-postgres && \ python3 -m pip install --no-cache -r requirements.txt && \ - rm -fr /usr/share/python-wheels/* + rm -fr /usr/share/python-wheels/* \ +RUN curl -sSL https://install.python-poetry.org | python3 - +RUN ln -sf /root/.local/bin/poetry /usr/local/bin FROM base as devlake-base diff --git a/backend/core/config/config_viper.go b/backend/core/config/config_viper.go index da3306b4e5d..85b3da07dfb 100644 --- a/backend/core/config/config_viper.go +++ b/backend/core/config/config_viper.go @@ -19,12 +19,13 @@ package config import ( "fmt" - "github.com/apache/incubator-devlake/core/errors" "os" "path/filepath" "regexp" "strings" + "github.com/apache/incubator-devlake/core/errors" + goerror "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" @@ -74,6 +75,8 @@ func setDefaultValue(v *viper.Viper) { v.SetDefault("PLUGIN_DIR", "bin/plugins") v.SetDefault("TEMPORAL_TASK_QUEUE", "DEVLAKE_TASK_QUEUE") v.SetDefault("TAP_PROPERTIES_DIR", "resources/tap") + v.SetDefault("ENABLE_REMOTE_PLUGINS", "true") + v.SetDefault("REMOTE_PLUGINS_STARTUP_PATH", "python/plugins/start.sh") } // replaceNewEnvItemInOldContent replace old config to new config in env file content diff --git a/backend/core/models/dynamic_tabler.go b/backend/core/models/dynamic_tabler.go new file mode 100644 index 00000000000..b7f92a7e7fc --- /dev/null +++ b/backend/core/models/dynamic_tabler.go @@ -0,0 +1,75 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" +) + +// DynamicTabler is a core.Tabler that wraps a runtime (anonymously) generated data-model. Due to limitations of +// reflection in Go and the GORM framework, the underlying model and the table have to be explicitly passed into dal.Dal's API +// via Unwrap() and TableName() +type DynamicTabler struct { + objType reflect.Type + wrapped any + table string +} + +func NewDynamicTabler(tableName string, objType reflect.Type) *DynamicTabler { + return &DynamicTabler{ + objType: objType, + table: tableName, + } +} + +func (d *DynamicTabler) New() *DynamicTabler { + return &DynamicTabler{ + objType: d.objType, + wrapped: reflect.New(d.objType).Interface(), + table: d.table, + } +} + +func (d *DynamicTabler) NewSlice() *DynamicTabler { + sliceType := reflect.SliceOf(d.objType) + return &DynamicTabler{ + objType: sliceType, + wrapped: reflect.New(sliceType).Interface(), + table: d.table, + } +} + +func (d *DynamicTabler) Unwrap() any { + return d.wrapped +} + +func (d *DynamicTabler) TableName() string { + return d.table +} + +var _ dal.Tabler = (*DynamicTabler)(nil) + +// UnwrapObject if the actual object is wrapped in some proxy, it unwinds and returns it, otherwise this is idempotent +func UnwrapObject(ifc any) any { + if dynamic, ok := ifc.(*DynamicTabler); ok { + return dynamic.Unwrap() + } + return ifc +} diff --git a/backend/core/utils/ipc.go b/backend/core/utils/ipc.go index 2c5d5b67eb6..cbd11c8bac6 100644 --- a/backend/core/utils/ipc.go +++ b/backend/core/utils/ipc.go @@ -20,97 +20,264 @@ package utils import ( "bufio" "fmt" - "github.com/apache/incubator-devlake/core/errors" + "io" "os" "os/exec" "strings" "sync" + "syscall" + + "github.com/apache/incubator-devlake/core/errors" ) +// NoopConverter no-op converter +var NoopConverter = func(b []byte) (any, errors.Error) { + return b, nil +} + // ProcessResponse wraps output of a process -type ProcessResponse[T any] struct { - Data T - Err error +type ProcessResponse struct { + stdout any + stderr any + fdOut any + err errors.Error } -// RunProcess runs the cmd and returns its raw standard output. This is a blocking function. -func RunProcess(cmd *exec.Cmd) (*ProcessResponse[[]byte], error) { - cmd.Env = append(cmd.Env, os.Environ()...) - stderr, err := cmd.StderrPipe() +// ProcessStream wraps output of a process +type ProcessStream struct { + receiveChannel <-chan *ProcessResponse + process *os.Process + cancelled bool +} + +// StreamProcessOptions options for streaming a process +type StreamProcessOptions struct { + OnStdout func(b []byte) (any, errors.Error) + OnStderr func(b []byte) (any, errors.Error) + // UseFdOut if true, it'll open this fd to be used by the child process. Useful to isolate stdout and custom outputs + UseFdOut bool + OnFdOut func(b []byte) (any, errors.Error) +} + +// RunProcessOptions options for running a process +type RunProcessOptions struct { + OnStdout func(b []byte) + OnStderr func(b []byte) + UseFdOut bool + OnFdOut func(b []byte) +} + +type processPipes struct { + stdout io.ReadCloser + stderr io.ReadCloser + fdOut io.ReadCloser +} + +func (p *processPipes) close() { + _ = p.stderr.Close() + _ = p.stdout.Close() + if p.fdOut != nil { + _ = p.fdOut.Close() + } +} + +// Receive listens to the process retrieval channel +func (p *ProcessStream) Receive() <-chan *ProcessResponse { + return p.receiveChannel +} + +// Cancel cancels the stream by sending a termination signal to the target. +func (p *ProcessStream) Cancel() errors.Error { + err := errors.Convert(p.process.Signal(syscall.SIGTERM)) + if err != nil { + return err + } + p.cancelled = true + return nil +} + +// GetStdout gets the stdout. The type depends on the conversion defined by StreamProcessOptions.OnStdout, otherwise it is []byte +func (resp *ProcessResponse) GetStdout() any { + return resp.stdout +} + +// GetStderr gets the stderr. The type depends on the conversion defined by StreamProcessOptions.OnStderr, otherwise it is []byte +func (resp *ProcessResponse) GetStderr() any { + return resp.stderr +} + +// GetFdOut gets the custom fd output. The type depends on the conversion defined by StreamProcessOptions.OnFdOut, otherwise it is []byte +func (resp *ProcessResponse) GetFdOut() any { + return resp.fdOut +} + +// GetError gets the error on the response +func (resp *ProcessResponse) GetError() errors.Error { + return resp.err +} + +// RunProcess runs the cmd and blocks until its completion. All returned results will have type []byte. +func RunProcess(cmd *exec.Cmd, opts *RunProcessOptions) (*ProcessResponse, errors.Error) { + stream, err := StreamProcess(cmd, &StreamProcessOptions{ + OnStdout: func(b []byte) (any, errors.Error) { + if opts.OnStdout != nil { + opts.OnStdout(b) + } + return NoopConverter(b) + }, + OnStderr: func(b []byte) (any, errors.Error) { + if opts.OnStderr != nil { + opts.OnStderr(b) + } + return NoopConverter(b) + }, + UseFdOut: opts.UseFdOut, + OnFdOut: func(b []byte) (any, errors.Error) { + if opts.OnFdOut != nil { + opts.OnFdOut(b) + } + return NoopConverter(b) + }, + }) if err != nil { return nil, err } - remoteErrorMsg := &strings.Builder{} - go func() { - scanner := bufio.NewScanner(stderr) - scanner.Split(bufio.ScanLines) - for scanner.Scan() { - _, _ = remoteErrorMsg.Write(scanner.Bytes()) - _, _ = remoteErrorMsg.WriteString("\n") + var stdout []byte + var stderr []byte + var fdOut []byte + for result := range stream.Receive() { + if result.err != nil { + err = result.err + break + } + if result.stdout != nil { + stdout = append(stdout, result.stdout.([]byte)...) + } + if result.stderr != nil { + stderr = append(stderr, result.stderr.([]byte)...) + } + if result.fdOut != nil { + fdOut = append(fdOut, result.fdOut.([]byte)...) } - }() - output, err := cmd.Output() - if err != nil { - return nil, errors.Default.Wrap(err, fmt.Sprintf("remote error message:\n%s", remoteErrorMsg.String())) } - return &ProcessResponse[[]byte]{ - Data: output, + return &ProcessResponse{ + stdout: stdout, + stderr: stderr, + fdOut: fdOut, + err: err, }, nil } -// StreamProcess runs the cmd and returns its standard output on a line-by-line basis, on a channel. The converter functor will allow you +// StreamProcess runs the cmd and returns its output on a line-by-line basis, on a channel. The converter functor will allow you // to convert the incoming raw to your custom data type T. This is a nonblocking function. -func StreamProcess[T any](cmd *exec.Cmd, converter func(b []byte) (T, error)) (<-chan *ProcessResponse[T], error) { - cmd.Env = append(cmd.Env, os.Environ()...) - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, err +func StreamProcess(cmd *exec.Cmd, opts *StreamProcessOptions) (*ProcessStream, errors.Error) { + if opts == nil { + opts = &StreamProcessOptions{} } - stderr, err := cmd.StderrPipe() + cmd.Env = append(cmd.Env, os.Environ()...) + pipes, err := getPipes(cmd, opts) if err != nil { return nil, err } - if err = cmd.Start(); err != nil { + if err = errors.Convert(cmd.Start()); err != nil { return nil, err } - stream := make(chan *ProcessResponse[T], 32) + receiveStream := make(chan *ProcessResponse, 32) wg := &sync.WaitGroup{} + stdScanner := scanOutputPipe(pipes.stdout, wg, opts.OnStdout, func(result any) *ProcessResponse { + return &ProcessResponse{stdout: result} + }, receiveStream) + errScanner, remoteErrorMsg := scanErrorPipe(pipes.stderr, receiveStream) + fdOutScanner := scanOutputPipe(pipes.fdOut, wg, opts.OnFdOut, func(result any) *ProcessResponse { + return &ProcessResponse{fdOut: result} + }, receiveStream) wg.Add(2) + if pipes.fdOut != nil { + wg.Add(1) + } + go stdScanner() + go errScanner() + if pipes.fdOut != nil { + go fdOutScanner() + } + processStream := &ProcessStream{ + process: cmd.Process, + receiveChannel: receiveStream, + } go func() { - scanner := bufio.NewScanner(stdout) + defer pipes.close() + if err = errors.Convert(cmd.Wait()); err != nil { + if !processStream.cancelled { + receiveStream <- &ProcessResponse{err: errors.Default.Wrap(err, fmt.Sprintf("remote error response:\n%s", remoteErrorMsg))} + } + } + wg.Done() + }() + go func() { + defer close(receiveStream) + wg.Wait() + }() + return processStream, nil +} + +func getPipes(cmd *exec.Cmd, opts *StreamProcessOptions) (*processPipes, errors.Error) { + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, errors.Convert(err) + } + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, errors.Convert(err) + } + var fdOut *os.File + if opts.UseFdOut { + fdReader, fdOutWriter, err := os.Pipe() + if err != nil { + return nil, errors.Convert(err) + } + cmd.ExtraFiles = []*os.File{fdOutWriter} + fdOut = fdReader + } + return &processPipes{ + stdout: stdout, + stderr: stderr, + fdOut: fdOut, + }, nil +} + +func scanOutputPipe(pipe io.ReadCloser, wg *sync.WaitGroup, onReceive func([]byte) (any, errors.Error), + responseCreator func(any) *ProcessResponse, outboundChannel chan<- *ProcessResponse) func() { + return func() { + scanner := bufio.NewScanner(pipe) scanner.Split(bufio.ScanLines) for scanner.Scan() { src := scanner.Bytes() data := make([]byte, len(src)) copy(data, src) - if result, err := converter(data); err != nil { - stream <- &ProcessResponse[T]{Err: err} + if result, err := onReceive(data); err != nil { + outboundChannel <- &ProcessResponse{err: err} } else { - stream <- &ProcessResponse[T]{Data: result} + outboundChannel <- responseCreator(result) } } wg.Done() - }() + } +} + +func scanErrorPipe(pipe io.ReadCloser, outboundChannel chan<- *ProcessResponse) (func(), *strings.Builder) { remoteErrorMsg := &strings.Builder{} - go func() { - scanner := bufio.NewScanner(stderr) + return func() { + scanner := bufio.NewScanner(pipe) scanner.Split(bufio.ScanLines) for scanner.Scan() { - _, _ = remoteErrorMsg.Write(scanner.Bytes()) + src := scanner.Bytes() + data := make([]byte, len(src)) + copy(data, src) + outboundChannel <- &ProcessResponse{stderr: data} + _, _ = remoteErrorMsg.Write(src) _, _ = remoteErrorMsg.WriteString("\n") } - }() - go func() { - if err = cmd.Wait(); err != nil { - stream <- &ProcessResponse[T]{Err: errors.Default.Wrap(err, fmt.Sprintf("remote error response:\n%s", remoteErrorMsg))} - } - wg.Done() - }() - go func() { - defer close(stream) - wg.Wait() - }() - return stream, nil + }, remoteErrorMsg } // CreateCmd wraps the args in "sh -c" for shell-level execution diff --git a/backend/go.mod b/backend/go.mod index c5f3e5b5bcc..6605d922778 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -3,6 +3,7 @@ module github.com/apache/incubator-devlake go 1.19 require ( + github.com/RaveNoX/go-jsonmerge v1.0.0 github.com/cockroachdb/errors v1.9.0 github.com/gin-contrib/cors v1.3.1 github.com/gin-gonic/gin v1.7.7 @@ -23,6 +24,7 @@ require ( github.com/robfig/cron/v3 v3.0.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/afero v1.6.0 + github.com/spf13/cast v1.4.1 github.com/spf13/cobra v1.5.0 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.8.0 @@ -108,7 +110,6 @@ require ( github.com/rogpeppe/go-internal v1.8.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect - github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.6-0.20200504143853-81378bbcd8a1 // indirect github.com/stretchr/objx v0.4.0 // indirect diff --git a/backend/go.sum b/backend/go.sum index 25f6dca46a6..5fea80b1492 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -58,6 +58,9 @@ github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C6 github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/RaveNoX/go-jsonmerge v1.0.0 h1:2e0nqnadoGUP8rAvcA0hkQelZreVO5X3BHomT2XMrAk= +github.com/RaveNoX/go-jsonmerge v1.0.0/go.mod h1:qYM/NA77LhO4h51JJM7Z+xBU3ovqrNIACZe+SkSNVFo= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= @@ -75,6 +78,7 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= @@ -445,6 +449,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= @@ -655,6 +660,7 @@ github.com/spf13/pflag v1.0.6-0.20200504143853-81378bbcd8a1/go.mod h1:McXfInJRrz github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= diff --git a/backend/helpers/pluginhelper/api/connection_helper.go b/backend/helpers/pluginhelper/api/connection_helper.go index 64760ed9536..da825d90cb5 100644 --- a/backend/helpers/pluginhelper/api/connection_helper.go +++ b/backend/helpers/pluginhelper/api/connection_helper.go @@ -18,6 +18,7 @@ limitations under the License. package api import ( + "github.com/apache/incubator-devlake/core/models" "strconv" "strings" @@ -93,40 +94,33 @@ func (c *ConnectionApiHelper) First(connection interface{}, params map[string]st // FirstById finds connection from db by id and decrypt it func (c *ConnectionApiHelper) FirstById(connection interface{}, id uint64) errors.Error { - err := c.db.First(connection, dal.Where("id = ?", id)) - if err != nil { - return err - } - return nil + return CallDB(c.db.First, connection, dal.Where("id = ?", id)) } // List returns all connections with password/token decrypted func (c *ConnectionApiHelper) List(connections interface{}) errors.Error { - err := c.db.All(connections) - if err != nil { - return err - } - return nil + return CallDB(c.db.All, connections) } // Delete connection func (c *ConnectionApiHelper) Delete(connection interface{}) errors.Error { - return c.db.Delete(connection) + return CallDB(c.db.Delete, connection) } func (c *ConnectionApiHelper) merge(connection interface{}, body map[string]interface{}) errors.Error { - if connectionValdiator, ok := connection.(apihelperabstract.ConnectionValidator); ok { + connection = models.UnwrapObject(connection) + if connectionValidator, ok := connection.(apihelperabstract.ConnectionValidator); ok { err := Decode(body, connection, nil) if err != nil { return err } - return connectionValdiator.ValidateConnection(connection, c.validator) + return connectionValidator.ValidateConnection(connection, c.validator) } return Decode(body, connection, c.validator) } func (c *ConnectionApiHelper) save(connection interface{}) errors.Error { - err := c.db.CreateOrUpdate(connection) + err := CallDB(c.db.CreateOrUpdate, connection) if err != nil { if strings.Contains(strings.ToLower(err.Error()), "duplicate") { return errors.BadInput.Wrap(err, "duplicated Connection Name") diff --git a/backend/helpers/pluginhelper/api/mapstructure.go b/backend/helpers/pluginhelper/api/mapstructure.go index 9a882e42c76..4ecbc9e6f9d 100644 --- a/backend/helpers/pluginhelper/api/mapstructure.go +++ b/backend/helpers/pluginhelper/api/mapstructure.go @@ -19,11 +19,13 @@ package api import ( "encoding/json" - "github.com/apache/incubator-devlake/core/errors" - "github.com/go-playground/validator/v10" "reflect" "time" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models" + "github.com/go-playground/validator/v10" + "github.com/mitchellh/mapstructure" ) @@ -79,7 +81,8 @@ func DecodeMapStruct(input map[string]interface{}, result interface{}) errors.Er // Decode decodes `source` into `target`. Pass an optional validator to validate the target. func Decode(source interface{}, target interface{}, vld *validator.Validate) errors.Error { - if err := mapstructure.Decode(source, target); err != nil { + target = models.UnwrapObject(target) + if err := mapstructure.Decode(source, &target); err != nil { return errors.Default.Wrap(err, "error decoding map into target type") } if vld != nil { diff --git a/backend/helpers/pluginhelper/api/misc_helpers.go b/backend/helpers/pluginhelper/api/misc_helpers.go new file mode 100644 index 00000000000..f5bfb24453b --- /dev/null +++ b/backend/helpers/pluginhelper/api/misc_helpers.go @@ -0,0 +1,33 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models" +) + +// CallDB wraps DB calls with this signature, and handles the case if the struct is wrapped in a models.DynamicTabler. +func CallDB(f func(any, ...dal.Clause) errors.Error, x any, clauses ...dal.Clause) errors.Error { + if dynamic, ok := x.(*models.DynamicTabler); ok { + clauses = append(clauses, dal.From(dynamic.TableName())) + x = dynamic.Unwrap() + } + return f(x, clauses...) +} diff --git a/backend/helpers/pluginhelper/tap/singer_tap_impl.go b/backend/helpers/pluginhelper/tap/singer_tap_impl.go index 98e62b4a46f..b86856b745f 100644 --- a/backend/helpers/pluginhelper/tap/singer_tap_impl.go +++ b/backend/helpers/pluginhelper/tap/singer_tap_impl.go @@ -19,6 +19,7 @@ package tap import ( "bufio" + "context" "encoding/json" "github.com/apache/incubator-devlake/core/config" "github.com/apache/incubator-devlake/core/errors" @@ -127,7 +128,7 @@ func (t *SingerTap) GetName() string { } // Run implements Tap.Run -func (t *SingerTap) Run() (<-chan *utils.ProcessResponse[Output[json.RawMessage]], errors.Error) { +func (t *SingerTap) Run(ctx context.Context) (<-chan *Response, errors.Error) { cmd := utils.CreateCmd( t.cmd, "--config", @@ -136,17 +137,32 @@ func (t *SingerTap) Run() (<-chan *utils.ProcessResponse[Output[json.RawMessage] t.propertiesFile.path, ifElse(t.stateFile.path != "", "--state "+t.stateFile.path, ""), ) - stream, err := utils.StreamProcess(cmd, func(b []byte) (Output[json.RawMessage], error) { - var output Output[json.RawMessage] - output, err := NewSingerTapOutput(b) - if err != nil { - return nil, err - } - return output, nil //data is expected to be JSON + rawStream, err := utils.StreamProcess(cmd, &utils.StreamProcessOptions{ + OnStdout: func(b []byte) (any, errors.Error) { + var output Output[json.RawMessage] + output, err := NewSingerTapOutput(b) + if err != nil { + return nil, err + } + return output, nil //data is expected to be JSON + }, }) if err != nil { return nil, errors.Default.Wrap(err, "error starting process stream from singer-tap") } + stream := make(chan *Response) + go func() { + defer close(stream) + for result := range rawStream.Receive() { + if err = result.GetError(); err != nil { + stream <- &Response{Err: err} + } + out := result.GetStdout() + if out != nil { + stream <- &Response{Out: out.(Output[json.RawMessage])} + } + } + }() return stream, nil } diff --git a/backend/helpers/pluginhelper/tap/tap.go b/backend/helpers/pluginhelper/tap/tap.go index 08fb9b54df3..9e94b754a31 100644 --- a/backend/helpers/pluginhelper/tap/tap.go +++ b/backend/helpers/pluginhelper/tap/tap.go @@ -18,15 +18,17 @@ limitations under the License. package tap import ( + "context" "encoding/json" + "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/utils" ) // Tap the abstract interface for Taps. Consumer code should not use concrete implementations directly. type Tap[Stream any] interface { - // Run runs the tap and returns a stream of results. Expected to be called after all the other Setters. - Run() (<-chan *utils.ProcessResponse[Output[json.RawMessage]], errors.Error) + // Run runs the tap and returns a stream of results. Expected to be called after all the other Setters. The Stdout of the response should + // be of type Output[json.RawMessage] + Run(ctx context.Context) (<-chan *Response, errors.Error) // GetName the name of this tap GetName() string // SetProperties Sets the properties of the tap and allows you to modify the properties at runtime. @@ -37,3 +39,8 @@ type Tap[Stream any] interface { // SetConfig sets the config of this tap SetConfig(config any) errors.Error } + +type Response struct { + Out Output[json.RawMessage] + Err errors.Error +} diff --git a/backend/helpers/pluginhelper/tap/tap_collector.go b/backend/helpers/pluginhelper/tap/tap_collector.go index 14ab2f53b87..37d2592615d 100644 --- a/backend/helpers/pluginhelper/tap/tap_collector.go +++ b/backend/helpers/pluginhelper/tap/tap_collector.go @@ -128,7 +128,7 @@ func (c *Collector[Stream]) Execute() (err errors.Error) { return err } } - resultStream, err := c.tapClient.Run() + resultStream, err := c.tapClient.Run(c.ctx.GetContext()) if err != nil { return err } @@ -156,7 +156,7 @@ func (c *Collector[Stream]) Execute() (err errors.Error) { return err default: } - output := result.Data + output := result.Out if tapRecord, ok := output.AsTapRecord(); ok { batchedResults = append(batchedResults, tapRecord.Record) c.ctx.IncProgress(1) diff --git a/backend/impls/dalgorm/encdec_serializer.go b/backend/impls/dalgorm/encdec_serializer.go index af3011107f4..ca140675c5d 100644 --- a/backend/impls/dalgorm/encdec_serializer.go +++ b/backend/impls/dalgorm/encdec_serializer.go @@ -21,9 +21,11 @@ import ( "context" "encoding/json" "fmt" - "github.com/apache/incubator-devlake/core/plugin" "reflect" + "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + "gorm.io/gorm/schema" ) @@ -37,6 +39,7 @@ type EncDecSerializer struct { // Scan implements serializer interface func (es *EncDecSerializer) Scan(ctx context.Context, field *schema.Field, dst reflect.Value, dbValue interface{}) (err error) { + dbValue = models.UnwrapObject(dbValue) fieldValue := reflect.New(field.FieldType) if dbValue != nil { var base64str string diff --git a/backend/python/build.sh b/backend/python/build.sh new file mode 100644 index 00000000000..f8d9a15d751 --- /dev/null +++ b/backend/python/build.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cd "${0%/*}" # make sure we're in the correct dir + +poetry config virtualenvs.create true + +for plugin_dir in $(ls -d plugins/*/*.toml); do + plugin_dir=$(dirname $plugin_dir) + echo "installing dependencies of python plugin in: $plugin_dir" &&\ + cd "$plugin_dir" &&\ + poetry install &&\ + cd - + exit_code=$? + if [ $exit_code != 0 ]; then + exit $exit_code + fi +done \ No newline at end of file diff --git a/backend/python/plugins/start.sh b/backend/python/plugins/start.sh new file mode 100755 index 00000000000..f5cdd9cf225 --- /dev/null +++ b/backend/python/plugins/start.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +endpoint="$1" + +cd "${0%/*}" # make sure we're in the correct dir + +for plugin_dir in $(ls -d */*.toml); do + plugin_dir=$(dirname $plugin_dir) + cd $plugin_dir &&\ + poetry run python $plugin_dir/main.py startup "$endpoint" &&\ + cd - + exit_code=$? + if [ $exit_code != 0 ]; then + exit $exit_code + fi +done \ No newline at end of file diff --git a/backend/python/pydevlake/README.md b/backend/python/pydevlake/README.md new file mode 100644 index 00000000000..ef675d30750 --- /dev/null +++ b/backend/python/pydevlake/README.md @@ -0,0 +1,334 @@ +# Pydevlake + +A framework to write data collection plugins for [DevLake](https://devlake.apache.org/). + + +# How to create a new plugin + +## Create plugin project + + +Make sure you have [Poetry](https://python-poetry.org/docs/#installation) installed. +Move to `python/plugins` and execute `poetry new myplugin`. +This will generate a new directory for your plugin. + +In the `pyproject.toml` file and add the following line at the end of the `[tool.poetry.dependencies]` section: +``` +pydevlake = { path = "../../pydevlake", develop = false } +``` + +Now run `poetry install`. + +## Create `main` file + +Create a `main.py` file with the following content: + +```python +from pydevlake import Plugin, Connection + + +class MyPluginConnection(Connection): + pass + + +class MyPlugin(Plugin): + @property + def connection_type(self): + return MyPluginConnection + + def test_connection(self, connection: MyPluginConnection): + pass + + @property + def streams(self): + return [] + + +if __name__ == '__main__': + MyPlugin.start() +``` + +This file is the entry point to your plugin. +It specifies three things: +- the parameters that your plugin needs to collect data, e.g. the url and credentials to connect to the datasource or custom options +- how to validate that some given parameters allows to connect to the datasource, e.g. test whether the credentials are correct +- the list of data streams that this plugin can collect + + +### Connection parameters + +The parameters of your plugin are defined as class attributes of the connection class. +For example to add a `url` parameter of type `str` edit `MyPLuginConnection` as follow: + +```python +class MyPluginConnection(Connection): + url: str +``` + + +## Add a new data stream + +A data stream groups the logic for: +- collecting the raw data from the datasource +- extracting this raw data into a tool-specific model +- converting the tool model into an equivalent [DevLake domain model](https://devlake.apache.org/docs/next/DataModels/DevLakeDomainLayerSchema) + + +### Create a tool model + +Create a `models.py` file. +Then create a class that modelizes the data your stream is going to collect. + +```python +from pydevlake.model import ToolModel + +class User(ToolModel, table=True): + id: str = Field(primary_key=True) + name: str + email: str +``` + +Your tool model must declare at least one attribute as a primary key, like `id` in the example above. +It must inherit from `ToolModel`, which in turn inherit from `SQLModel`, the base class of an [ORM of the same name](https://sqlmodel.tiangolo.com/). +You can use `SQLModel` features like [declaring relationships with other models](https://sqlmodel.tiangolo.com/tutorial/relationship-attributes/). + + +### Create the stream class + +Create a new file for your first stream in a `streams` directory. + +```python +from pydevlake import Stream +from pydevlake.domain_layer.crossdomain import User as DomainUser + +from myplugin.models import User as ToolUser + + +class Users(Stream): + tool_model = ToolUser + domain_model = DomainUser + + def collect(self, state, context) -> Iterable[Tuple[object, dict]]: + pass + + def convert(self, user: ToolUser) -> Iterable[DomainUser]: + pass +``` + +This stream will collect raw user data, e.g. as parsed JSON objects, extract this raw data as your +tool-specific user model, then convert them into domain-layer user models. + +It is possible to have a stream that produce several domain models of different types from a single tool model. In that case, declare the list of possible domain model types in a class attribute `domain_models` (plural) instead of a single domain model type in the `domain_model` (singular) class attribute. + +The `collect` method takes a `state` dictionary and a context object and yields tuples of raw data and new state. +The last state that the plugin yielded for a given connection will be reused during the next collection. +The plugin can use this `state` to store information necessary to perform incremental collection of data. + + +The `convert` method takes a tool-specific user model and convert it into domain level user models. +Here the two models align quite well, the conversion is trivial: + +```python +def convert(self, user: ToolUser) -> Iterable[DomainUser]: + yield DomainUser( + id=user.id, + name=user.name + email=user.email + ) +``` + +### Create an API wrapper + +Lets assume that your datasource is a REST API. +We can create the following class to define it. + +```python +from pydevlake.api import API + + +class MyAPI(API): + def __init__(self, url: str): + self.url = url + + def users(self): + return self.get(f'{self.url}/users') +``` + +By inheriting `API` you get access to facilities to wrap REST APIs. +Here the `users` method will return a `Response` object that contains the results of calling `GET` on `/users`. + +Now we can go back to our stream file and implement `collect`: + +```python +from myplugin.api import MyAPI + +... + + def collect(self, state, context) -> Iterable[Tuple[object, dict]]: + api = MyAPI(context.connection.url) + for user in api.users().json(): + yield user, state + +... +``` + +If the API responds with a list of JSON object with properties matching your `User` model attributes, you're done!. +Indeed extraction has a default implementation that takes of this common case. +This is why it is important to make tool models that align with the data you collect. + +If this is not the case, e.g. the attribute case mismatch, you can redefine the `extract` method: + +```python +... + +class Users(Stream): + ... + + def extract(self, raw_data: dict) -> ToolModel: + return ToolUser( + id=raw_data["ID"], + name=raw_data["Name"], + email=raw_data["Email"] + ) + + ... +``` + + +#### Request and response hook + +For each request sent and response received by your API wrapper, +you can register hooks. Hooks allows you to implement +authentication, pagination, and generic API error handling. + +For example, lets assume that we are dealing with an API that +require user to authenticate via a token set in a request header. + +A request hook is declared in your API with a `@request_hook` decorator. + +```python +... +class MyAPI(API): + def __init__(self, url, token): + self.url = url + self.token = token + + ... + @request_hook + def authenticate(self, request): + if self.token: + request.headers['Token'] = self.token + ... +``` + +Here the method `authenticate` is a hook that is run on each request. +Similarly you can declare response hooks with `@response_hook`. + +#### Pagination + +One usage of a response hook is for handling paginated results. +A response hook can be used to wrap the `Response` object in a +`PagedResponse` object that support iteration and fetching other pages. +This response hook is actually defined in `API` base class and expect +your API wrapper to declare a `paginator` property. + +You can subclass `Paginator` to provide API specific logic or reuse an +existing implementation such as `TokenPaginator`. +A token paginator assumes the API paginated responses are JSON object with one +property that is an array of items and another that contains the token to the next +page. + +For example, the following paginator fetch items from the `'results'` attribute, +the next page from the `'nextPage'` attribute and will issue requests with a `page` +query parameter. + +``` +... +class MyAPI(API): + ... + paginator = TokenPaginator( + items_attr='results', + next_page_token_attr='nextPage', + next_page_token_param='page' + ) + ... +``` + +## Substreams + +With REST APIs, you often need to fetch a stream of items, and then to collect additional +data for each of those items. + +For example you might want to collect all `UserComments` written by each user collected via the `Users` stream. + +To handle such cases, `UserComments` would inherit from `Substream` instead of `Stream`. +A substream needs to specify which is his parent stream. The `collect` method +of a substream will be called with each item collected from the parent stream. + +```python +... +from pydevlake import Substream +from myplugin.streams.users import Users + +class UserComments(Substream): + parent_stream = Users # Must specify the parent stream + ... + def collect(self, state: dict, context, user: User): + """ + This method will be called for each user collected from parent stream Users. + """ + for json in MyPluginAPI(context.connection.token).user_comments(user.id): + yield json, state + ... +``` + + +# Test the plugin standalone + +To test your plugin manually, you can run your `main.py` file with different commands. +You can find all those commands with `--help` cli flag: + +```console +poetry run myplugin/main.py --help +``` + +For testing, the interesting commands are `collect`/`extract`/`convert`. +Each takes a context and a stream name. +The context is a JSON object that must at least contain: +- a `connection_id` +- a `db_url`, e.g. you can use `"sqlite+pysqlite:///:memory:"` for an in-memory DB +- a `connection` object containing the same attributes than your plugin connection type + +Also, python plugins communicate with go side over an extra file descriptor 3, so you should +redirect to stdout when testing your plugin. + +``` +console +CTX='{"connection_id": "1", "db_url":"sqlite+pysqlite:///:memory:", "connection": {...your connection attrs here...}}' +poetry run myplugin/main.py $CTX users 3>&1 +``` + + +# Test the plugin with DevLake + +To test your plugin together with DevLake, you first need to create a connection for your plugin and get its id. +One easy way to do that is to run DevLake with `make dev` and then to create the connection with a POST +request to your plugin connection API: + +```console +curl -X 'POST' \ + 'http://localhost:8080/plugins/myplugin/connections' \ + -d '{...connection JSON object...}' +``` + +You should get the created connection with his id (which is 1 for the first created connection) in the response. + +Now that a connection for your plugin exists in DevLake database, we can try to run your plugin using `backend/server/services/remote/run/run.go` script: + +```console +cd backend +go run server/services/remote/run/run.go -c 1 -p python/plugins/myplugin/myplugin/main.py +``` + +This script takes a connection id (`-c` flag) and the path to your plugin `main.py` file (`-p` flag). +You can also send options as a JSON object (`-o` flag). diff --git a/backend/python/pydevlake/poetry.lock b/backend/python/pydevlake/poetry.lock new file mode 100644 index 00000000000..7c89577ec9b --- /dev/null +++ b/backend/python/pydevlake/poetry.lock @@ -0,0 +1,624 @@ +# This file is automatically @generated by Poetry and should not be changed by hand. + +[[package]] +name = "attrs" +version = "22.2.0" +description = "Classes Without Boilerplate" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, + {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] +tests = ["attrs[tests-no-zope]", "zope.interface"] +tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990)", "mypy (>=0.971,<0.990)", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-mypy-plugins", "pytest-xdist[psutil]", "pytest-xdist[psutil]"] + +[[package]] +name = "certifi" +version = "2022.12.7" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.0.1" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "charset-normalizer-3.0.1.tar.gz", hash = "sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db72b07027db150f468fbada4d85b3b2729a3db39178abf5c543b784c1254539"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62595ab75873d50d57323a91dd03e6966eb79c41fa834b7a1661ed043b2d404d"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:772b87914ff1152b92a197ef4ea40efe27a378606c39446ded52c8f80f79702e"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70990b9c51340e4044cfc394a81f614f3f90d41397104d226f21e66de668730d"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:292d5e8ba896bbfd6334b096e34bffb56161c81408d6d036a7dfa6929cff8783"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2edb64ee7bf1ed524a1da60cdcd2e1f6e2b4f66ef7c077680739f1641f62f555"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:31a9ddf4718d10ae04d9b18801bd776693487cbb57d74cc3458a7673f6f34639"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:44ba614de5361b3e5278e1241fda3dc1838deed864b50a10d7ce92983797fa76"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:12db3b2c533c23ab812c2b25934f60383361f8a376ae272665f8e48b88e8e1c6"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c512accbd6ff0270939b9ac214b84fb5ada5f0409c44298361b2f5e13f9aed9e"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-win32.whl", hash = "sha256:502218f52498a36d6bf5ea77081844017bf7982cdbe521ad85e64cabee1b608b"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:601f36512f9e28f029d9481bdaf8e89e5148ac5d89cffd3b05cd533eeb423b59"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0298eafff88c99982a4cf66ba2efa1128e4ddaca0b05eec4c456bbc7db691d8d"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8d0fc946c784ff7f7c3742310cc8a57c5c6dc31631269876a88b809dbeff3d3"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:87701167f2a5c930b403e9756fab1d31d4d4da52856143b609e30a1ce7160f3c"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14e76c0f23218b8f46c4d87018ca2e441535aed3632ca134b10239dfb6dadd6b"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c0a590235ccd933d9892c627dec5bc7511ce6ad6c1011fdf5b11363022746c1"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c7fe7afa480e3e82eed58e0ca89f751cd14d767638e2550c77a92a9e749c317"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79909e27e8e4fcc9db4addea88aa63f6423ebb171db091fb4373e3312cb6d603"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ac7b6a045b814cf0c47f3623d21ebd88b3e8cf216a14790b455ea7ff0135d18"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:72966d1b297c741541ca8cf1223ff262a6febe52481af742036a0b296e35fa5a"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5995f0164fa7df59db4746112fec3f49c461dd6b31b841873443bdb077c13cfc"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4a8fcf28c05c1f6d7e177a9a46a1c52798bfe2ad80681d275b10dcf317deaf0b"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:761e8904c07ad053d285670f36dd94e1b6ab7f16ce62b9805c475b7aa1cffde6"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-win32.whl", hash = "sha256:71140351489970dfe5e60fc621ada3e0f41104a5eddaca47a7acb3c1b851d6d3"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ab77acb98eba3fd2a85cd160851816bfce6871d944d885febf012713f06659c"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:84c3990934bae40ea69a82034912ffe5a62c60bbf6ec5bc9691419641d7d5c9a"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74292fc76c905c0ef095fe11e188a32ebd03bc38f3f3e9bcb85e4e6db177b7ea"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c95a03c79bbe30eec3ec2b7f076074f4281526724c8685a42872974ef4d36b72"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c39b0e3eac288fedc2b43055cfc2ca7a60362d0e5e87a637beac5d801ef478"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df2c707231459e8a4028eabcd3cfc827befd635b3ef72eada84ab13b52e1574d"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93ad6d87ac18e2a90b0fe89df7c65263b9a99a0eb98f0a3d2e079f12a0735837"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:59e5686dd847347e55dffcc191a96622f016bc0ad89105e24c14e0d6305acbc6"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:cd6056167405314a4dc3c173943f11249fa0f1b204f8b51ed4bde1a9cd1834dc"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:083c8d17153ecb403e5e1eb76a7ef4babfc2c48d58899c98fcaa04833e7a2f9a"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:f5057856d21e7586765171eac8b9fc3f7d44ef39425f85dbcccb13b3ebea806c"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:7eb33a30d75562222b64f569c642ff3dc6689e09adda43a082208397f016c39a"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:95dea361dd73757c6f1c0a1480ac499952c16ac83f7f5f4f84f0658a01b8ef41"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:eaa379fcd227ca235d04152ca6704c7cb55564116f8bc52545ff357628e10602"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3e45867f1f2ab0711d60c6c71746ac53537f1684baa699f4f668d4c6f6ce8e14"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cadaeaba78750d58d3cc6ac4d1fd867da6fc73c88156b7a3212a3cd4819d679d"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:911d8a40b2bef5b8bbae2e36a0b103f142ac53557ab421dc16ac4aafee6f53dc"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:503e65837c71b875ecdd733877d852adbc465bd82c768a067badd953bf1bc5a3"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a60332922359f920193b1d4826953c507a877b523b2395ad7bc716ddd386d866"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16a8663d6e281208d78806dbe14ee9903715361cf81f6d4309944e4d1e59ac5b"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a16418ecf1329f71df119e8a65f3aa68004a3f9383821edcb20f0702934d8087"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9d9153257a3f70d5f69edf2325357251ed20f772b12e593f3b3377b5f78e7ef8"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:02a51034802cbf38db3f89c66fb5d2ec57e6fe7ef2f4a44d070a593c3688667b"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:2e396d70bc4ef5325b72b593a72c8979999aa52fb8bcf03f701c1b03e1166918"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:11b53acf2411c3b09e6af37e4b9005cba376c872503c8f28218c7243582df45d"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:0bf2dae5291758b6f84cf923bfaa285632816007db0330002fa1de38bfcb7154"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2c03cc56021a4bd59be889c2b9257dae13bf55041a3372d3295416f86b295fb5"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:024e606be3ed92216e2b6952ed859d86b4cfa52cd5bc5f050e7dc28f9b43ec42"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4b0d02d7102dd0f997580b51edc4cebcf2ab6397a7edf89f1c73b586c614272c"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:358a7c4cb8ba9b46c453b1dd8d9e431452d5249072e4f56cfda3149f6ab1405e"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81d6741ab457d14fdedc215516665050f3822d3e56508921cc7239f8c8e66a58"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8b8af03d2e37866d023ad0ddea594edefc31e827fee64f8de5611a1dbc373174"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9cf4e8ad252f7c38dd1f676b46514f92dc0ebeb0db5552f5f403509705e24753"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e696f0dd336161fca9adbb846875d40752e6eba585843c768935ba5c9960722b"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c22d3fe05ce11d3671297dc8973267daa0f938b93ec716e12e0f6dee81591dc1"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:109487860ef6a328f3eec66f2bf78b0b72400280d8f8ea05f69c51644ba6521a"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:37f8febc8ec50c14f3ec9637505f28e58d4f66752207ea177c1d67df25da5aed"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f97e83fa6c25693c7a35de154681fcc257c1c41b38beb0304b9c4d2d9e164479"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a152f5f33d64a6be73f1d30c9cc82dfc73cec6477ec268e7c6e4c7d23c2d2291"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:39049da0ffb96c8cbb65cbf5c5f3ca3168990adf3551bd1dee10c48fce8ae820"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-win32.whl", hash = "sha256:4457ea6774b5611f4bed5eaa5df55f70abde42364d498c5134b7ef4c6958e20e"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:e62164b50f84e20601c1ff8eb55620d2ad25fb81b59e3cd776a1902527a788af"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8eade758719add78ec36dc13201483f8e9b5d940329285edcd5f70c0a9edbd7f"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8499ca8f4502af841f68135133d8258f7b32a53a1d594aa98cc52013fff55678"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3fc1c4a2ffd64890aebdb3f97e1278b0cc72579a08ca4de8cd2c04799a3a22be"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00d3ffdaafe92a5dc603cb9bd5111aaa36dfa187c8285c543be562e61b755f6b"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2ac1b08635a8cd4e0cbeaf6f5e922085908d48eb05d44c5ae9eabab148512ca"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6f45710b4459401609ebebdbcfb34515da4fc2aa886f95107f556ac69a9147e"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ae1de54a77dc0d6d5fcf623290af4266412a7c4be0b1ff7444394f03f5c54e3"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b590df687e3c5ee0deef9fc8c547d81986d9a1b56073d82de008744452d6541"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab5de034a886f616a5668aa5d098af2b5385ed70142090e2a31bcbd0af0fdb3d"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9cb3032517f1627cc012dbc80a8ec976ae76d93ea2b5feaa9d2a5b8882597579"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:608862a7bf6957f2333fc54ab4399e405baad0163dc9f8d99cb236816db169d4"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0f438ae3532723fb6ead77e7c604be7c8374094ef4ee2c5e03a3a17f1fca256c"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:356541bf4381fa35856dafa6a965916e54bed415ad8a24ee6de6e37deccf2786"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-win32.whl", hash = "sha256:39cf9ed17fe3b1bc81f33c9ceb6ce67683ee7526e65fde1447c772afc54a1bb8"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59"}, + {file = "charset_normalizer-3.0.1-py3-none-any.whl", hash = "sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.1.0" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, + {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "fire" +version = "0.4.0" +description = "A library for automatically generating command line interfaces." +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "fire-0.4.0.tar.gz", hash = "sha256:c5e2b8763699d1142393a46d0e3e790c5eb2f0706082df8f647878842c216a62"}, +] + +[package.dependencies] +six = "*" +termcolor = "*" + +[[package]] +name = "greenlet" +version = "2.0.2" +description = "Lightweight in-process concurrent programming" +category = "main" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" +files = [ + {file = "greenlet-2.0.2-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:bdfea8c661e80d3c1c99ad7c3ff74e6e87184895bbaca6ee8cc61209f8b9b85d"}, + {file = "greenlet-2.0.2-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:9d14b83fab60d5e8abe587d51c75b252bcc21683f24699ada8fb275d7712f5a9"}, + {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, + {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, + {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, + {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, + {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, + {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, + {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d75209eed723105f9596807495d58d10b3470fa6732dd6756595e89925ce2470"}, + {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3a51c9751078733d88e013587b108f1b7a1fb106d402fb390740f002b6f6551a"}, + {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, + {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, + {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, + {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, + {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, + {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, + {file = "greenlet-2.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:eff4eb9b7eb3e4d0cae3d28c283dc16d9bed6b193c2e1ace3ed86ce48ea8df19"}, + {file = "greenlet-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5454276c07d27a740c5892f4907c86327b632127dd9abec42ee62e12427ff7e3"}, + {file = "greenlet-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:7cafd1208fdbe93b67c7086876f061f660cfddc44f404279c1585bbf3cdc64c5"}, + {file = "greenlet-2.0.2-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:910841381caba4f744a44bf81bfd573c94e10b3045ee00de0cbf436fe50673a6"}, + {file = "greenlet-2.0.2-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:18a7f18b82b52ee85322d7a7874e676f34ab319b9f8cce5de06067384aa8ff43"}, + {file = "greenlet-2.0.2-cp35-cp35m-win32.whl", hash = "sha256:03a8f4f3430c3b3ff8d10a2a86028c660355ab637cee9333d63d66b56f09d52a"}, + {file = "greenlet-2.0.2-cp35-cp35m-win_amd64.whl", hash = "sha256:4b58adb399c4d61d912c4c331984d60eb66565175cdf4a34792cd9600f21b394"}, + {file = "greenlet-2.0.2-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:703f18f3fda276b9a916f0934d2fb6d989bf0b4fb5a64825260eb9bfd52d78f0"}, + {file = "greenlet-2.0.2-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:32e5b64b148966d9cccc2c8d35a671409e45f195864560829f395a54226408d3"}, + {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dd11f291565a81d71dab10b7033395b7a3a5456e637cf997a6f33ebdf06f8db"}, + {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e0f72c9ddb8cd28532185f54cc1453f2c16fb417a08b53a855c4e6a418edd099"}, + {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd021c754b162c0fb55ad5d6b9d960db667faad0fa2ff25bb6e1301b0b6e6a75"}, + {file = "greenlet-2.0.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:3c9b12575734155d0c09d6c3e10dbd81665d5c18e1a7c6597df72fd05990c8cf"}, + {file = "greenlet-2.0.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b9ec052b06a0524f0e35bd8790686a1da006bd911dd1ef7d50b77bfbad74e292"}, + {file = "greenlet-2.0.2-cp36-cp36m-win32.whl", hash = "sha256:dbfcfc0218093a19c252ca8eb9aee3d29cfdcb586df21049b9d777fd32c14fd9"}, + {file = "greenlet-2.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:9f35ec95538f50292f6d8f2c9c9f8a3c6540bbfec21c9e5b4b751e0a7c20864f"}, + {file = "greenlet-2.0.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:d5508f0b173e6aa47273bdc0a0b5ba055b59662ba7c7ee5119528f466585526b"}, + {file = "greenlet-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1"}, + {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9c59a2120b55788e800d82dfa99b9e156ff8f2227f07c5e3012a45a399620b7"}, + {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2780572ec463d44c1d3ae850239508dbeb9fed38e294c68d19a24d925d9223ca"}, + {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:937e9020b514ceedb9c830c55d5c9872abc90f4b5862f89c0887033ae33c6f73"}, + {file = "greenlet-2.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:36abbf031e1c0f79dd5d596bfaf8e921c41df2bdf54ee1eed921ce1f52999a86"}, + {file = "greenlet-2.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:18e98fb3de7dba1c0a852731c3070cf022d14f0d68b4c87a19cc1016f3bb8b33"}, + {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, + {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, + {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, + {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, + {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, + {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, + {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acd2162a36d3de67ee896c43effcd5ee3de247eb00354db411feb025aa319857"}, + {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0bf60faf0bc2468089bdc5edd10555bab6e85152191df713e2ab1fcc86382b5a"}, + {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, + {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, + {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, + {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, + {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, + {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, + {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be4ed120b52ae4d974aa40215fcdfde9194d63541c7ded40ee12eb4dda57b76b"}, + {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94c817e84245513926588caf1152e3b559ff794d505555211ca041f032abbb6b"}, + {file = "greenlet-2.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1a819eef4b0e0b96bb0d98d797bef17dc1b4a10e8d7446be32d1da33e095dbb8"}, + {file = "greenlet-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7efde645ca1cc441d6dc4b48c0f7101e8d86b54c8530141b09fd31cef5149ec9"}, + {file = "greenlet-2.0.2-cp39-cp39-win32.whl", hash = "sha256:ea9872c80c132f4663822dd2a08d404073a5a9b5ba6155bea72fb2a79d1093b5"}, + {file = "greenlet-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:db1a39669102a1d8d12b57de2bb7e2ec9066a6f2b3da35ae511ff93b01b5d564"}, + {file = "greenlet-2.0.2.tar.gz", hash = "sha256:e7c8dc13af7db097bed64a051d2dd49e9f0af495c26995c00a9ee842690d34c0"}, +] + +[package.extras] +docs = ["Sphinx", "docutils (<0.18)"] +test = ["objgraph", "psutil"] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] + +[[package]] +name = "inflect" +version = "6.0.2" +description = "Correctly generate plurals, singular nouns, ordinals, indefinite articles; convert numbers to words" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "inflect-6.0.2-py3-none-any.whl", hash = "sha256:182741ec7e9e4c8f7f55b01fa6d80bcd3c4a183d349dfa6d9abbff0a1279e98f"}, + {file = "inflect-6.0.2.tar.gz", hash = "sha256:f1a6bcb0105046f89619fde1a7d044c612c614c2d85ef182582d9dc9b86d309a"}, +] + +[package.dependencies] +pydantic = ">=1.9.1" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "pygments", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "mysqlclient" +version = "2.1.1" +description = "Python interface to MySQL" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ + {file = "mysqlclient-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:c1ed71bd6244993b526113cca3df66428609f90e4652f37eb51c33496d478b37"}, + {file = "mysqlclient-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:c812b67e90082a840efb82a8978369e6e69fc62ce1bda4ca8f3084a9d862308b"}, + {file = "mysqlclient-2.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:0d1cd3a5a4d28c222fa199002810e8146cffd821410b67851af4cc80aeccd97c"}, + {file = "mysqlclient-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:b355c8b5a7d58f2e909acdbb050858390ee1b0e13672ae759e5e784110022994"}, + {file = "mysqlclient-2.1.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:996924f3483fd36a34a5812210c69e71dea5a3d5978d01199b78b7f6d485c855"}, + {file = "mysqlclient-2.1.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:dea88c8d3f5a5d9293dfe7f087c16dd350ceb175f2f6631c9cf4caf3e19b7a96"}, + {file = "mysqlclient-2.1.1.tar.gz", hash = "sha256:828757e419fb11dd6c5ed2576ec92c3efaa93a0f7c39e263586d1ee779c3d782"}, +] + +[[package]] +name = "packaging" +version = "23.0" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, + {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, +] + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pydantic" +version = "1.10.4" +description = "Data validation and settings management using python type hints" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5635de53e6686fe7a44b5cf25fcc419a0d5e5c1a1efe73d49d48fe7586db854"}, + {file = "pydantic-1.10.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6dc1cc241440ed7ca9ab59d9929075445da6b7c94ced281b3dd4cfe6c8cff817"}, + {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51bdeb10d2db0f288e71d49c9cefa609bca271720ecd0c58009bd7504a0c464c"}, + {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78cec42b95dbb500a1f7120bdf95c401f6abb616bbe8785ef09887306792e66e"}, + {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8775d4ef5e7299a2f4699501077a0defdaac5b6c4321173bcb0f3c496fbadf85"}, + {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:572066051eeac73d23f95ba9a71349c42a3e05999d0ee1572b7860235b850cc6"}, + {file = "pydantic-1.10.4-cp310-cp310-win_amd64.whl", hash = "sha256:7feb6a2d401f4d6863050f58325b8d99c1e56f4512d98b11ac64ad1751dc647d"}, + {file = "pydantic-1.10.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39f4a73e5342b25c2959529f07f026ef58147249f9b7431e1ba8414a36761f53"}, + {file = "pydantic-1.10.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:983e720704431a6573d626b00662eb78a07148c9115129f9b4351091ec95ecc3"}, + {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d52162fe6b2b55964fbb0af2ee58e99791a3138588c482572bb6087953113a"}, + {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fdf8d759ef326962b4678d89e275ffc55b7ce59d917d9f72233762061fd04a2d"}, + {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05a81b006be15655b2a1bae5faa4280cf7c81d0e09fcb49b342ebf826abe5a72"}, + {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d88c4c0e5c5dfd05092a4b271282ef0588e5f4aaf345778056fc5259ba098857"}, + {file = "pydantic-1.10.4-cp311-cp311-win_amd64.whl", hash = "sha256:6a05a9db1ef5be0fe63e988f9617ca2551013f55000289c671f71ec16f4985e3"}, + {file = "pydantic-1.10.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:887ca463c3bc47103c123bc06919c86720e80e1214aab79e9b779cda0ff92a00"}, + {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdf88ab63c3ee282c76d652fc86518aacb737ff35796023fae56a65ced1a5978"}, + {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a48f1953c4a1d9bd0b5167ac50da9a79f6072c63c4cef4cf2a3736994903583e"}, + {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a9f2de23bec87ff306aef658384b02aa7c32389766af3c5dee9ce33e80222dfa"}, + {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:cd8702c5142afda03dc2b1ee6bc358b62b3735b2cce53fc77b31ca9f728e4bc8"}, + {file = "pydantic-1.10.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6e7124d6855b2780611d9f5e1e145e86667eaa3bd9459192c8dc1a097f5e9903"}, + {file = "pydantic-1.10.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b53e1d41e97063d51a02821b80538053ee4608b9a181c1005441f1673c55423"}, + {file = "pydantic-1.10.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:55b1625899acd33229c4352ce0ae54038529b412bd51c4915349b49ca575258f"}, + {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301d626a59edbe5dfb48fcae245896379a450d04baeed50ef40d8199f2733b06"}, + {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6f9d649892a6f54a39ed56b8dfd5e08b5f3be5f893da430bed76975f3735d15"}, + {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d7b5a3821225f5c43496c324b0d6875fde910a1c2933d726a743ce328fbb2a8c"}, + {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f2f7eb6273dd12472d7f218e1fef6f7c7c2f00ac2e1ecde4db8824c457300416"}, + {file = "pydantic-1.10.4-cp38-cp38-win_amd64.whl", hash = "sha256:4b05697738e7d2040696b0a66d9f0a10bec0efa1883ca75ee9e55baf511909d6"}, + {file = "pydantic-1.10.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a9a6747cac06c2beb466064dda999a13176b23535e4c496c9d48e6406f92d42d"}, + {file = "pydantic-1.10.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb992a1ef739cc7b543576337bebfc62c0e6567434e522e97291b251a41dad7f"}, + {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:990406d226dea0e8f25f643b370224771878142155b879784ce89f633541a024"}, + {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e82a6d37a95e0b1b42b82ab340ada3963aea1317fd7f888bb6b9dfbf4fff57c"}, + {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9193d4f4ee8feca58bc56c8306bcb820f5c7905fd919e0750acdeeeef0615b28"}, + {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2b3ce5f16deb45c472dde1a0ee05619298c864a20cded09c4edd820e1454129f"}, + {file = "pydantic-1.10.4-cp39-cp39-win_amd64.whl", hash = "sha256:9cbdc268a62d9a98c56e2452d6c41c0263d64a2009aac69246486f01b4f594c4"}, + {file = "pydantic-1.10.4-py3-none-any.whl", hash = "sha256:4948f264678c703f3877d1c8877c4e3b2e12e549c57795107f08cf70c6ec7774"}, + {file = "pydantic-1.10.4.tar.gz", hash = "sha256:b9a3859f24eb4e097502a3be1fb4b2abb79b6103dd9e2e0edb70613a4459a648"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pydevd-pycharm" +version = "231.6471.3" +description = "PyCharm Debugger (used in PyCharm and PyDev)" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "pydevd-pycharm-231.6471.3.tar.gz", hash = "sha256:3c77340fd1f7e984790081e0a9e89e0284450ce242df7d01aecabbd24e4e46ae"}, +] + +[[package]] +name = "pytest" +version = "7.2.1" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"}, + {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"}, +] + +[package.dependencies] +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "requests" +version = "2.28.2" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7, <4" +files = [ + {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, + {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "sqlalchemy" +version = "1.4.41" +description = "Database Abstraction Library" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "SQLAlchemy-1.4.41-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:13e397a9371ecd25573a7b90bd037db604331cf403f5318038c46ee44908c44d"}, + {file = "SQLAlchemy-1.4.41-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2d6495f84c4fd11584f34e62f9feec81bf373787b3942270487074e35cbe5330"}, + {file = "SQLAlchemy-1.4.41-cp27-cp27m-win32.whl", hash = "sha256:e570cfc40a29d6ad46c9aeaddbdcee687880940a3a327f2c668dd0e4ef0a441d"}, + {file = "SQLAlchemy-1.4.41-cp27-cp27m-win_amd64.whl", hash = "sha256:5facb7fd6fa8a7353bbe88b95695e555338fb038ad19ceb29c82d94f62775a05"}, + {file = "SQLAlchemy-1.4.41-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:f37fa70d95658763254941ddd30ecb23fc4ec0c5a788a7c21034fc2305dab7cc"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:361f6b5e3f659e3c56ea3518cf85fbdae1b9e788ade0219a67eeaaea8a4e4d2a"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0990932f7cca97fece8017414f57fdd80db506a045869d7ddf2dda1d7cf69ecc"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cd767cf5d7252b1c88fcfb58426a32d7bd14a7e4942497e15b68ff5d822b41ad"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5102fb9ee2c258a2218281adcb3e1918b793c51d6c2b4666ce38c35101bb940e"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-win32.whl", hash = "sha256:2082a2d2fca363a3ce21cfa3d068c5a1ce4bf720cf6497fb3a9fc643a8ee4ddd"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-win_amd64.whl", hash = "sha256:e4b12e3d88a8fffd0b4ca559f6d4957ed91bd4c0613a4e13846ab8729dc5c251"}, + {file = "SQLAlchemy-1.4.41-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:90484a2b00baedad361402c257895b13faa3f01780f18f4a104a2f5c413e4536"}, + {file = "SQLAlchemy-1.4.41-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b67fc780cfe2b306180e56daaa411dd3186bf979d50a6a7c2a5b5036575cbdbb"}, + {file = "SQLAlchemy-1.4.41-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ad2b727fc41c7f8757098903f85fafb4bf587ca6605f82d9bf5604bd9c7cded"}, + {file = "SQLAlchemy-1.4.41-cp311-cp311-win32.whl", hash = "sha256:59bdc291165b6119fc6cdbc287c36f7f2859e6051dd923bdf47b4c55fd2f8bd0"}, + {file = "SQLAlchemy-1.4.41-cp311-cp311-win_amd64.whl", hash = "sha256:d2e054aed4645f9b755db85bc69fc4ed2c9020c19c8027976f66576b906a74f1"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:4ba7e122510bbc07258dc42be6ed45997efdf38129bde3e3f12649be70683546"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0dcf127bb99458a9d211e6e1f0f3edb96c874dd12f2503d4d8e4f1fd103790b"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e16c2be5cb19e2c08da7bd3a87fed2a0d4e90065ee553a940c4fc1a0fb1ab72b"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5ebeeec5c14533221eb30bad716bc1fd32f509196318fb9caa7002c4a364e4c"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-win32.whl", hash = "sha256:3e2ef592ac3693c65210f8b53d0edcf9f4405925adcfc031ff495e8d18169682"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-win_amd64.whl", hash = "sha256:eb30cf008850c0a26b72bd1b9be6730830165ce049d239cfdccd906f2685f892"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:c23d64a0b28fc78c96289ffbd0d9d1abd48d267269b27f2d34e430ea73ce4b26"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8eb8897367a21b578b26f5713833836f886817ee2ffba1177d446fa3f77e67c8"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:14576238a5f89bcf504c5f0a388d0ca78df61fb42cb2af0efe239dc965d4f5c9"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:639e1ae8d48b3c86ffe59c0daa9a02e2bfe17ca3d2b41611b30a0073937d4497"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-win32.whl", hash = "sha256:0005bd73026cd239fc1e8ccdf54db58b6193be9a02b3f0c5983808f84862c767"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-win_amd64.whl", hash = "sha256:5323252be2bd261e0aa3f33cb3a64c45d76829989fa3ce90652838397d84197d"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:05f0de3a1dc3810a776275763764bb0015a02ae0f698a794646ebc5fb06fad33"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0002e829142b2af00b4eaa26c51728f3ea68235f232a2e72a9508a3116bd6ed0"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:22ff16cedab5b16a0db79f1bc99e46a6ddececb60c396562e50aab58ddb2871c"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccfd238f766a5bb5ee5545a62dd03f316ac67966a6a658efb63eeff8158a4bbf"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-win32.whl", hash = "sha256:58bb65b3274b0c8a02cea9f91d6f44d0da79abc993b33bdedbfec98c8440175a"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-win_amd64.whl", hash = "sha256:ce8feaa52c1640de9541eeaaa8b5fb632d9d66249c947bb0d89dd01f87c7c288"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:199a73c31ac8ea59937cc0bf3dfc04392e81afe2ec8a74f26f489d268867846c"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4676d51c9f6f6226ae8f26dc83ec291c088fe7633269757d333978df78d931ab"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:036d8472356e1d5f096c5e0e1a7e0f9182140ada3602f8fff6b7329e9e7cfbcd"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2307495d9e0ea00d0c726be97a5b96615035854972cc538f6e7eaed23a35886c"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-win32.whl", hash = "sha256:9c56e19780cd1344fcd362fd6265a15f48aa8d365996a37fab1495cae8fcd97d"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-win_amd64.whl", hash = "sha256:f5fa526d027d804b1f85cdda1eb091f70bde6fb7d87892f6dd5a48925bc88898"}, + {file = "SQLAlchemy-1.4.41.tar.gz", hash = "sha256:0292f70d1797e3c54e862e6f30ae474014648bc9c723e14a2fda730adb0a9791"}, +] + +[package.dependencies] +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} + +[package.extras] +aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] +asyncio = ["greenlet (!=0.4.17)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"] +mssql = ["pyodbc"] +mssql-pymssql = ["pymssql"] +mssql-pyodbc = ["pyodbc"] +mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] +mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] +mysql-connector = ["mysql-connector-python"] +oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"] +postgresql = ["psycopg2 (>=2.7)"] +postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] +postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] +postgresql-psycopg2binary = ["psycopg2-binary"] +postgresql-psycopg2cffi = ["psycopg2cffi"] +pymysql = ["pymysql", "pymysql (<1)"] +sqlcipher = ["sqlcipher3_binary"] + +[[package]] +name = "sqlalchemy2-stubs" +version = "0.0.2a32" +description = "Typing Stubs for SQLAlchemy 1.4" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "sqlalchemy2-stubs-0.0.2a32.tar.gz", hash = "sha256:2a2cfab71d35ac63bf21ad841d8610cd93a3bd4c6562848c538fa975585c2739"}, + {file = "sqlalchemy2_stubs-0.0.2a32-py3-none-any.whl", hash = "sha256:7f5fb30b0cf7c6b74c50c1d94df77ff32007afee8d80499752eb3fedffdbdfb8"}, +] + +[package.dependencies] +typing-extensions = ">=3.7.4" + +[[package]] +name = "sqlmodel" +version = "0.0.8" +description = "SQLModel, SQL databases in Python, designed for simplicity, compatibility, and robustness." +category = "main" +optional = false +python-versions = ">=3.6.1,<4.0.0" +files = [ + {file = "sqlmodel-0.0.8-py3-none-any.whl", hash = "sha256:0fd805719e0c5d4f22be32eb3ffc856eca3f7f20e8c7aa3e117ad91684b518ee"}, + {file = "sqlmodel-0.0.8.tar.gz", hash = "sha256:3371b4d1ad59d2ffd0c530582c2140b6c06b090b32af9b9c6412986d7b117036"}, +] + +[package.dependencies] +pydantic = ">=1.8.2,<2.0.0" +SQLAlchemy = ">=1.4.17,<=1.4.41" +sqlalchemy2-stubs = "*" + +[[package]] +name = "termcolor" +version = "2.2.0" +description = "ANSI color formatting for output in terminal" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "termcolor-2.2.0-py3-none-any.whl", hash = "sha256:91ddd848e7251200eac969846cbae2dacd7d71c2871e92733289e7e3666f48e7"}, + {file = "termcolor-2.2.0.tar.gz", hash = "sha256:dfc8ac3f350788f23b2947b3e6cfa5a53b630b612e6cd8965a015a776020b99a"}, +] + +[package.extras] +tests = ["pytest", "pytest-cov"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.4.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, +] + +[[package]] +name = "urllib3" +version = "1.26.14" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, + {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "e4293f87235f9c63a69951ac8ea600dd64719fa7f28c059decad45974adc5222" diff --git a/backend/python/pydevlake/pydevlake/__init__.py b/backend/python/pydevlake/pydevlake/__init__.py new file mode 100644 index 00000000000..edabf45f07d --- /dev/null +++ b/backend/python/pydevlake/pydevlake/__init__.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .model import ToolModel +from .logger import logger +from .message import Connection +from .plugin import Plugin +from .stream import Stream, Substream +from .context import Context diff --git a/backend/python/pydevlake/pydevlake/api.py b/backend/python/pydevlake/pydevlake/api.py new file mode 100644 index 00000000000..cc2a930cd7d --- /dev/null +++ b/backend/python/pydevlake/pydevlake/api.py @@ -0,0 +1,290 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +from typing import Optional, Union +import json + +import requests as req +from urllib.parse import urljoin + + +RouteArgs = Union[list[str], dict[str, str]] +QueryArgs = dict[str, str] +Headers = dict[str, str] + + +class Request: + def __init__(self, + url: str, + query_args: Optional[QueryArgs] = None, + headers: Optional[Headers] = None): + self.url = url + self.query_args = query_args or {} + self.headers = headers or {} + + def copy(self): + return Request(self.url, self.query_args, self.headers) + + def __str__(self): + if self.query_args: + query_str = '&'.join(f'{k}={v}' for k, v in self.query_args.items()) + return f'{self.url}?{query_str}' + return self.url + + +class Response: + def __init__(self, + request: Request, + status: int, + body: bytes = None, + headers: Headers = None): + self.request = request + self.status = status + self.body = body or bytes() + self.headers = headers or {} + + @property + def json(self): + if not hasattr(self, '_json'): + self._json = json.loads(self.body) + return self._json + + def __str__(self): + return f'{self.request}: {self.status}' + + +# Sentinel value to abort processing of requests/responses in hooks +ABORT = object() + + +class APIBase: + """ + The base class for defining APIs. + It implements a hook system to preprocess requests before sending them and postprocess response + before returning them. + Hooks are declared by decorating methods with `@request_hook` and `@response_hook`. + Hooks are executed in the order they are declared. + """ + @property + def session(self): + if not hasattr(self, '_session'): + self._session = req.Session() + return self._session + + @property + def base_url(self): + pass + + def send(self, request: Request): + request = self._apply_hooks(request, self.request_hooks()) + if request is ABORT: + return ABORT + + res = self.session.get( + url=request.url, + headers=request.headers, + params=request.query_args + ) + + response = Response( + request=request, + status=res.status_code, + body=res.content, + headers=res.headers + ) + + response = self._apply_hooks(response, self.response_hooks()) + + return response + + def _apply_hooks(self, target, hooks): + for hook in hooks: + result = hook.apply(target, self) + if result is ABORT: + return ABORT + + if isinstance(result, type(target)): + target = result + return target + + def get(self, path, **query_args): + req = Request(urljoin(self.base_url, path), query_args) + return self.send(req) + + def request_hooks(self): + if not hasattr(self, '_request_hooks'): + self._request_hooks = [h for h in self._iter_members() if isinstance(h, RequestHook)] + return self._request_hooks + + def response_hooks(self): + if not hasattr(self, '_response_hooks'): + self._response_hooks = [h for h in self._iter_members() if isinstance(h, ResponseHook)] + return self._response_hooks + + def _iter_members(self): + for c in reversed(type(self).__mro__): + for m in c.__dict__.values(): + yield m + + +class RequestHook: + """ + Preprocess a request before sending it. + """ + def apply(self, request: Request, api: APIBase): + pass + + +class CustomRequestHook(RequestHook): + def __init__(self, fn): + self.fn = fn + + def apply(self, request: Request, api: APIBase): + return self.fn(api, request) + +request_hook = CustomRequestHook + + +class ResponseHook: + def apply(self, response: Response, api: APIBase): + pass + + +class CustomResponseHook(ResponseHook): + def __init__(self, fn): + self.fn = fn + + def apply(self, response: Response, api: APIBase): + return self.fn(api, response) + +response_hook = CustomResponseHook + + +class Paginator: + """ + Encapsulate logic for handling paginated responses. + """ + def get_items(self, response) -> Optional[list[object]]: + """ + Extracts the items from a response, e.g. returning the + `items` attribute of a JSON body. + Returning None indicates that the response is not paginated. + """ + pass + + def get_next_page_id(self, response) -> Optional[int | str]: + """ + Extracts or compute the id of the next page from the response, + e.g. incrementing the value of `page` of a JSON body. + This id will be suplied to the next request via `set_next_page_param`. + Returning None indicates that the response is the last page. + """ + pass + + def set_next_page_param(self, request, next_page_id: int | str): + """ + Modify the request to set the parameter for fetching next page, + e.g. set the `page` query parameter. + """ + pass + + +class PagedResponse(Response): + """ + Decorate requests.Response to add iteration of items + within the page and fetching next pages. + """ + def __init__(self, response, paginator, api): + self.response = response + self.paginator = paginator + self.api = api + + def __iter__(self): + current = self.response + + while True: + items = self.paginator.get_items(current) + for item in items: + yield item + + next_page_id = self.paginator.get_next_page_id(current) + if not next_page_id: + # No next page + return + + # Fetch next page + next_request = current.request.copy() + self.paginator.set_next_page_param(next_request, next_page_id) + current = self.api.send(next_request) + + def __getattr__(self, attr_name): + # Delegate everything to Response + return getattr(self.response, attr_name) + + +class TokenPaginator(Paginator): + def __init__(self, items_attr: str, next_page_token_attr: str, next_page_token_param: str): + self.items_attr = items_attr + self.next_page_token_attr = next_page_token_attr + self.next_page_token_param = next_page_token_param + + def get_items(self, response) -> Optional[list[object]]: + return response.json[self.items_attr] + + def get_next_page_id(self, response) -> Optional[int | str]: + return response.json[self.next_page_token_attr] + + def set_next_page_param(self, request, next_page_id): + request.query_args[self.next_page_token_param] = next_page_id + + +class APIException(Exception): + def __init__(self, response): + self.response = response + + def __str__(self): + return f'APIException: {self.response}' + + +class API(APIBase): + """ + Provides hooks for: + - pagination: define the `paginator` property in subclasses + + # TODO: + - Error handling response hook: retries, + - Rate limitation + """ + @property + def paginator(self) -> Paginator: + """ + Redefine in subclass to handle pagination + """ + return None + + @response_hook + def handle_error(self, response): + if response.status != 200: + raise APIException(response) + + @response_hook + def paginate(self, response): + paginator = self.paginator + if not paginator: + return + return PagedResponse(response, paginator, self) diff --git a/backend/python/pydevlake/pydevlake/context.py b/backend/python/pydevlake/pydevlake/context.py new file mode 100644 index 00000000000..a5c97b5ff63 --- /dev/null +++ b/backend/python/pydevlake/pydevlake/context.py @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from urllib.parse import urlparse, parse_qsl +from sqlmodel import SQLModel, create_engine + +from pydevlake.message import Connection + + +class Context: + def __init__(self, db_url: str, connection_id: int, connection: Connection, options: dict): + self.db_url = db_url + self.connection_id = connection_id + self.connection = connection + self.options = options + self._engine = None + + @property + def engine(self): + if not self._engine: + db_url = self.db_url + if not db_url: + raise Exception("Missing db_url setting") + + # `parseTime` parameter is not understood by MySQL driver + # so we have to parse query args to remove it + connect_args = dict(parse_qsl(urlparse(self.db_url).query)) + db_url = self.db_url.split('?')[0] + + if 'parseTime' in connect_args: + del connect_args['parseTime'] + + self._engine = create_engine(db_url, connect_args=connect_args) + SQLModel.metadata.create_all(self._engine) + return self._engine + + @property + def incremental(self): + return self.options.get('incremental') is True diff --git a/backend/python/pydevlake/pydevlake/doc.template.json b/backend/python/pydevlake/pydevlake/doc.template.json new file mode 100644 index 00000000000..b152a5cf615 --- /dev/null +++ b/backend/python/pydevlake/pydevlake/doc.template.json @@ -0,0 +1,155 @@ +{ + "info": { + "title": "$plugin_name plugin documentation", + "version": "1.0.0" + }, + "openapi": "3.0.2", + "paths": { + "/plugins/$plugin_name/connections/{connectionId}": { + "get": { + "description": "Get a connection", + "parameters": [ + { + "$$ref": "#/components/parameters/connectionId" + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$$ref": "#/components/schemas/connection" + } + } + } + }, + "400": { + "description": "There is no connection with given id" + } + } + }, + "patch": { + "description": "Update a connection", + "parameters": [ + { + "$$ref": "#/components/parameters/connectionId" + }, + { + "name": "connection", + "required": true, + "in": "body", + "schema": { + "$$ref": "#/components/schemas/connection" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$$ref": "#/components/schemas/connection" + } + } + } + } + } + }, + "delete": { + "description": "Delete a connection", + "parameters": [ + { + "$$ref": "#/components/parameters/connectionId" + } + ], + "response": { + "200": { + "description": "The connection has been deleted" + }, + "400": { + "description": "There is no connection with given id" + } + } + } + }, + "/plugins/$plugin_name/connections": { + "get": { + "description": "Get all connections", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$$ref": "#/components/schemas/connection" + } + } + } + } + } + } + }, + "post": { + "description": "Create a connection", + "parameters": [ + { + "name": "connection", + "required": true, + "in": "body", + "schema": { + "$$ref": "#/components/schemas/connection" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$$ref": "#/components/schemas/connection" + } + } + } + } + } + } + }, + "/plugins/$plugin_name/test": { + "post": { + "description": "Test if a connection is valid", + "body": { + "application/json": { + "schema": { + "$$ref": "#/components/schemas/connection" + } + } + } + }, + "response": { + "200": { + "description": "The connection is valid" + }, + "400": { + "description": "The connection is not valid" + } + } + } + }, + "components": { + "schemas": { + "connection": $connection_schema + }, + "parameters": { + "connectionId": { + "name": "connectionId", + "description": "Id of the connection", + "in": "path", + "required": true, + "schema": { + "type": "int" + } + } + } + } +} diff --git a/backend/python/pydevlake/pydevlake/docgen.py b/backend/python/pydevlake/pydevlake/docgen.py new file mode 100644 index 00000000000..acce0a39fb9 --- /dev/null +++ b/backend/python/pydevlake/pydevlake/docgen.py @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Type +from pathlib import Path +from string import Template +import json + +from pydevlake.message import Connection + + +# TODO: Move swagger documentation generation to GO side along with API implementation +TEMPLATE_PATH = str(Path(__file__).parent / 'doc.template.json') + +def generate_doc(plugin_name: str, connection_type: Type[Connection]): + with open(TEMPLATE_PATH, 'r') as f: + doc_template = Template(f.read()) + connection_schema = connection_type.schema_json() + doc = doc_template.substitute(plugin_name=plugin_name, connection_schema=connection_schema) + return json.loads(doc) diff --git a/backend/python/pydevlake/pydevlake/domain_layer/__init__.py b/backend/python/pydevlake/pydevlake/domain_layer/__init__.py new file mode 100644 index 00000000000..65d64ce95c0 --- /dev/null +++ b/backend/python/pydevlake/pydevlake/domain_layer/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py b/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py new file mode 100644 index 00000000000..b001531df8d --- /dev/null +++ b/backend/python/pydevlake/pydevlake/domain_layer/crossdomain.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from pydevlake.model import DomainModel + + +class User(DomainModel, table=True): + __table_name__ = 'users' + + name: str + email: str diff --git a/backend/python/pydevlake/pydevlake/domain_layer/devops.py b/backend/python/pydevlake/pydevlake/domain_layer/devops.py new file mode 100644 index 00000000000..c15dd5c5d40 --- /dev/null +++ b/backend/python/pydevlake/pydevlake/domain_layer/devops.py @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Optional +from datetime import datetime +from enum import Enum + +from sqlmodel import Field, Relationship + +from pydevlake.model import DomainModel, NoPKModel + + +class CICDScope(DomainModel): + name: str + description: str + url: str + createdDate: datetime + updatedDate: datetime + + +class CICDPipeline(DomainModel, table=True): + __table_name__ = 'cicd_pipelines' + + class Result(Enum): + SUCCESS = "SUCCESS" + FAILURE = "FAILURE" + ABORT = "ABORT" + MANUAL = "MANUAL" + + class Status(Enum): + IN_PROGRESS = "IN_PROGRESS" + DONE = "DONE" + MANUAL = "MANUAL" + + class Type(Enum): + CI = "CI" + CD = "CD" + + name: str + status: Status + created_date: datetime + finished_date: Optional[datetime] + result: Optional[Result] + duration_sec: Optional[int] + environment: Optional[str] + type: Optional[Type] #Unused + + # parent_pipelines: list["CICDPipeline"] = Relationship(back_populates="child_pipelines", link_model="CICDPipelineRelationship") + # child_pipelines: list["CICDPipeline"] = Relationship(back_populates="parent_pipelines", link_model="CICDPipelineRelationship") + + +class CICDPipelineRelationship(NoPKModel): + __table_name__ = 'cicd_pipeline_relationships' + parent_pipeline_id: str = Field(primary_key=True, foreign_key=CICDPipeline.id) + child_pipeline_id: str = Field(primary_key=True, foreign_key=CICDPipeline.id) diff --git a/backend/python/pydevlake/pydevlake/extractor.py b/backend/python/pydevlake/pydevlake/extractor.py new file mode 100644 index 00000000000..8ee42914a43 --- /dev/null +++ b/backend/python/pydevlake/pydevlake/extractor.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Type +from pydevlake import ToolModel + + +def autoextract(json: dict, model_cls: Type[ToolModel]) -> ToolModel: + annotations = dict(model_cls.__annotations__) + for key, value in json.items(): + if key in annotations: + expected_type = annotations[key] + if isinstance(expected_type, type) and issubclass(expected_type, ToolModel): + # TODO: replace with actual foreign key + json[key] = value["id"] + return model_cls(**json) diff --git a/backend/python/pydevlake/pydevlake/ipc.py b/backend/python/pydevlake/pydevlake/ipc.py new file mode 100644 index 00000000000..a457b979461 --- /dev/null +++ b/backend/python/pydevlake/pydevlake/ipc.py @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from functools import wraps +from typing import Generator, TextIO + +from pydevlake.context import Context +from pydevlake.message import Message + + +def plugin_method(func): + def open_send_channel() -> TextIO: + fd = 3 + return os.fdopen(fd, 'w') + + def send_output(send_ch: TextIO, obj: object): + if not isinstance(obj, Message): + return + send_ch.write(obj.json()) + send_ch.write('\n') + send_ch.flush() + + @wraps(func) + def wrapper(self, *args): + ret = func(self, *args) + if ret is not None: + with open_send_channel() as send_ch: + if isinstance(ret, Generator): + for each in ret: + send_output(send_ch, each) + else: + send_output(send_ch, ret) + return None + + return wrapper + + +class PluginCommands: + def __init__(self, plugin): + self._plugin = plugin + + @plugin_method + def collect(self, ctx: dict, stream: str): + yield from self._plugin.collect(self._mk_context(ctx), stream) + + @plugin_method + def extract(self, ctx: dict, stream: str): + yield from self._plugin.extract(self._mk_context(ctx), stream) + + @plugin_method + def convert(self, ctx: dict, stream: str): + yield from self._plugin.convert(self._mk_context(ctx), stream) + + @plugin_method + def test_connection(self, connection: dict): + connection = self._plugin.connection_type(**connection) + self._plugin.test_connection(connection) + + @plugin_method + def make_pipeline(self, ctx: dict, scopes: list[dict]): + yield from self._plugin.make_pipeline(self._mk_context(ctx), scopes) + + @plugin_method + def run_migrations(self, force: bool): + self._plugin.run_migrations(force) + + @plugin_method + def plugin_info(self): + return self._plugin.plugin_info() + + def startup(self, endpoint: str): + self._plugin.startup(endpoint) + + def _mk_context(self, data: dict): + db_url = data['db_url'] + connection_id = data['connection_id'] + connection = self._plugin.connection_type(**data['connection']) + options = data.get('options', {}) + return Context(db_url, connection_id, connection, options) diff --git a/backend/python/pydevlake/pydevlake/logger.py b/backend/python/pydevlake/pydevlake/logger.py new file mode 100644 index 00000000000..8f297ab8ea6 --- /dev/null +++ b/backend/python/pydevlake/pydevlake/logger.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import sys +import logging + + +stdout_handler = logging.StreamHandler(sys.stdout) +stdout_handler.addFilter(lambda rec: rec.levelno < logging.ERROR) + + +stderr_handler = logging.StreamHandler(sys.stderr) +stderr_handler.addFilter(lambda rec: rec.levelno >= logging.ERROR) + +logging.basicConfig( + level=logging.DEBUG, + format='%(levelname)s: %(message)s', + handlers=[stdout_handler, stderr_handler] +) + +logger = logging.getLogger() + + +if __name__ == "__main__": + logger.info('hey') + logger.debug('wut?') + logger.error('oops') + logger.critical('boom') diff --git a/backend/python/pydevlake/pydevlake/message.py b/backend/python/pydevlake/pydevlake/message.py new file mode 100644 index 00000000000..de09167c47b --- /dev/null +++ b/backend/python/pydevlake/pydevlake/message.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from pydantic import BaseModel + + +class Message(BaseModel): + pass + + +class SubtaskMeta(BaseModel): + name: str + entry_point_name: str + required: bool + enabled_by_default: bool + description: str + domain_types: list[str] + arguments: list[str] = None + + +class PluginInfo(Message): + name: str + description: str + connection_schema: dict + plugin_path: str + subtask_metas: list[SubtaskMeta] + extension: str = "datasource" + type: str = "python-poetry" + + +class SwaggerDoc(Message): + name: str + resource: str + spec: dict + + +class PluginDetails(Message): + plugin_info: PluginInfo + swagger: SwaggerDoc + + +class RemoteProgress(Message): + increment: int = 0 + current: int = 0 + total: int = 0 + + +class Connection(Message): + pass + + +class PipelineTask(Message): + plugin: str + # Do not snake_case this attribute, + # it must match the json tag name in PipelineTask go struct + skipOnFail: bool + subtasks: list[str] + options: dict[str, object] + + +class PipelineStage(Message): + tasks: list[PipelineTask] + + +class PipelinePlan(Message): + stages: list[PipelineStage] + + +class PipelineScope(Message): + id: str + name: str + table_name: str + + +class BlueprintScope(Message): + id: str + name: str diff --git a/backend/python/pydevlake/pydevlake/model.py b/backend/python/pydevlake/pydevlake/model.py new file mode 100644 index 00000000000..3c870501000 --- /dev/null +++ b/backend/python/pydevlake/pydevlake/model.py @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from typing import Optional +from inspect import getmodule + +from datetime import datetime +from sqlalchemy import Column, String, DateTime, func +from sqlalchemy.orm import declared_attr +from sqlalchemy.inspection import inspect +from sqlmodel import SQLModel, Field +import inflect + +inflect_engine = inflect.engine() + + +def get_plugin_name(cls): + """ + Get the plugin name from a class by looking into + the file path of its module. + """ + module = getmodule(cls) + path_segments = module.__file__.split(os.sep) + # Finds the name of the first enclosing folder + # that is not a python module + depth = len(module.__name__.split('.')) + 1 + return path_segments[-depth] + + +class RawModel(SQLModel): + id: int = Field(primary_key=True) + params: str = b'' + data: bytes + url: str = b'' + input: bytes = b'' + created_at: datetime = Field(default_factory=datetime.now) + + +class RawDataOrigin(SQLModel): + # SQLModel doesn't like attributes starting with _ + # so we change the names of the columns. + raw_data_params: str = Field(sa_column_kwargs={'name':'_raw_data_params'}) + raw_data_table: str = Field(sa_column_kwargs={'name':'_raw_data_table'}) + raw_data_id: Optional[str] = Field(sa_column_kwargs={'name':'_raw_data_id'}) + raw_data_remark: Optional[str] = Field(sa_column_kwargs={'name':'_raw_data_remark'}) + + def set_origin(self, raw: RawModel): + self.raw_data_id = raw.id + self.raw_data_params = raw.params + self.raw_data_table = raw.__tablename__ + + +class ToolModel(RawDataOrigin): + @declared_attr + def __tablename__(cls) -> str: + plugin_name = get_plugin_name(cls) + plural_entity = inflect_engine.plural_noun(cls.__name__.lower()) + return f'_tool_{plugin_name}_{plural_entity}' + + +class NoPKModel(SQLModel): + created_at: datetime = Field( + sa_column=Column(DateTime(), default=func.now()) + ) + updated_at: datetime = Field( + sa_column=Column(DateTime(), default=func.now(), onupdate=func.now()) + ) + + +class DomainModel(NoPKModel): + id: str = Field(primary_key=True) + + +def generate_domain_id(tool_model: ToolModel, connection_id: str): + """ + Generate an identifier for a domain entity + from the tool entity it originates from. + """ + model_type = type(tool_model) + segments = [get_plugin_name(model_type), model_type.__name__, str(connection_id)] + mapper = inspect(model_type) + for primary_key_column in mapper.primary_key: + prop = mapper.get_property_by_column(primary_key_column) + attr_val = getattr(tool_model, prop.key) + segments.append(str(attr_val)) + return ':'.join(segments) diff --git a/backend/python/pydevlake/pydevlake/plugin.py b/backend/python/pydevlake/pydevlake/plugin.py new file mode 100644 index 00000000000..0e3c0a166a0 --- /dev/null +++ b/backend/python/pydevlake/pydevlake/plugin.py @@ -0,0 +1,174 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Type, Union, Iterable +import sys +from abc import abstractmethod +import requests + +import fire + +import pydevlake.message as msg +from pydevlake.subtasks import Subtask +from pydevlake.docgen import generate_doc +from pydevlake.ipc import PluginCommands +from pydevlake.context import Context +from pydevlake.stream import Stream +from pydevlake.model import DomainModel + + +class Plugin: + def __init__(self): + self._streams = dict() + for stream in self.streams: + if isinstance(stream, type): + stream = stream(self.name) + self._streams[stream.name] = stream + + @property + def name(self) -> str: + """ + The name of the plugin, defaults to the class name lowercased. + """ + return type(self).__name__.lower().removesuffix('plugin') + + @property + def description(self) -> str: + return f"{self.name} plugin" + + @property + @abstractmethod + def connection_type(self) -> Type[msg.Connection]: + pass + + @abstractmethod + def test_connection(self, connection: msg.Connection): + """ + Test if the the connection with the datasource can be established with the given connection. + Must raise an exception if the connection can't be established. + """ + pass + + @property + def subtasks(self) -> list[Subtask]: + return [subtask for stream in self._streams.values() for subtask in stream.subtasks] + + @abstractmethod + def get_scopes(self, scope_name: str, connection: msg.Connection) -> Iterable[DomainModel]: + pass + + @property + def streams(self) -> list[Union[Stream, Type[Stream]]]: + pass + + def collect(self, ctx: Context, stream: str): + yield from self.get_stream(stream).collector.run(ctx) + + def extract(self, ctx: Context, stream: str): + yield from self.get_stream(stream).extractor.run(ctx) + + def convert(self, ctx: Context, stream: str): + yield from self.get_stream(stream).convertor.run(ctx) + + def run_migrations(self, force: bool): + # TODO: Create tables + pass + + def make_pipeline(self, ctx: Context, scopes: list[msg.BlueprintScope]): + """ + Make a simple pipeline using the scopes declared by the plugin. + """ + stages = [ + msg.PipelineStage( + tasks=[ + msg.PipelineTask( + self.name, + skipOnFail=False, + subtasks=[t.name for t in self.subtasks], + options={ + "scopeId": scope.id, + "scopeName": scope.name} + ) + ] + ) + for scope in scopes + ] + + plan = msg.PipelinePlan(stages=stages) + yield plan + + scopes = [ + msg.PipelineScope( + id=':'.join([self.name, type(scope).__name__, ctx.connection_id, bp_scope.id]), + name=bp_scope.name, + table_name=scope.__tablename__ + ) + for bp_scope in scopes + for scope in self.get_scopes(bp_scope.name, ctx.connection) + ] + yield scopes + + def get_stream(self, stream_name: str): + stream = self._streams.get(stream_name) + if stream is None: + raise Exception(f'Unkown stream {stream_name}') + return stream + + def startup(self, endpoint: str): + details = msg.PluginDetails( + plugin_info=self.plugin_info(), + swagger=msg.SwaggerDoc( + name=self.name, + resource=self.name, + spec=generate_doc(self.name, self.connection_type) + ) + ) + resp = requests.post(f"{endpoint}/plugins/register", data=details.json()) + if resp.status_code != 200: + raise Exception(f"unexpected http status code {resp.status_code}: {resp.content}") + + def plugin_info(self): + subtask_metas = [ + msg.SubtaskMeta( + name=subtask.name, + entry_point_name=subtask.verb, + arguments=[subtask.stream.name], + required=True, + enabled_by_default=True, + description=subtask.description, + domain_types=[dm.__name__ for dm in subtask.stream.domain_models] + ) + for subtask in self.subtasks + ] + + return msg.PluginInfo( + name=self.name, + description=self.description, + plugin_path=self._plugin_path(), + extension="datasource", + connection_schema=self.connection_type.schema(), + subtask_metas=subtask_metas + ) + + def _plugin_path(self): + module_name = type(self).__module__ + module = sys.modules[module_name] + return module.__file__ + + @classmethod + def start(cls): + plugin = cls() + fire.Fire(PluginCommands(plugin)) diff --git a/backend/python/pydevlake/pydevlake/stream.py b/backend/python/pydevlake/pydevlake/stream.py new file mode 100644 index 00000000000..7422bd0de55 --- /dev/null +++ b/backend/python/pydevlake/pydevlake/stream.py @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Iterable, Type +from abc import abstractmethod + +from pydevlake.subtasks import Collector, Extractor, Convertor, SubstreamCollector +from pydevlake.model import RawModel, ToolModel, DomainModel + + +class Stream: + def __init__(self, plugin_name: str): + self.plugin_name = plugin_name + self.collector = Collector(self) + self.extractor = Extractor(self) + self.convertor = Convertor(self) + self._raw_model = None + + @property + def subtasks(self): + return [self.collector, self.extractor, self.convertor] + + @property + def name(self): + return type(self).__name__.lower() + + @property + def qualified_name(self): + return f'{self.plugin_name}_{self.name}' + + @property + def tool_model(self) -> Type[ToolModel]: + pass + + @property + def domain_model(self) -> Type[DomainModel]: + pass + + @property + def domain_models(self) -> Type[DomainModel]: + assert self.domain_model, "Streams must declare their domain_model or domain_models" + return [self.domain_model] + + def raw_model(self, session) -> Type[RawModel]: + if self._raw_model is not None: + return self._raw_model + + table_name = f'_raw_{self.plugin_name}_{self.name}' + + # Look for existing raw model + for mapper in RawModel._sa_registry.mappers: + model = mapper.class_ + if model.__tablename__ == table_name: + self._raw_model = model + return self._raw_model + + # Create raw model + class StreamRawModel(RawModel, table=True): + __tablename__ = table_name + + self._raw_model = StreamRawModel + RawModel.metadata.create_all(session.get_bind()) + return self._raw_model + + def collect(self, state, context) -> Iterable[tuple[object, dict]]: + pass + + def extract(self, raw_data: dict) -> ToolModel: + return self.tool_model(**raw_data) + + def convert(self, tool_model: ToolModel) -> DomainModel: + pass + + +class Substream(Stream): + def __init__(self, plugin_name: str): + super().__init__(plugin_name) + self.collector = SubstreamCollector(self) + + @property + @abstractmethod + def parent_stream(self): + pass + + def collect(self, state, context, parent) -> Iterable[tuple[object, dict]]: + pass diff --git a/backend/python/pydevlake/pydevlake/subtasks.py b/backend/python/pydevlake/pydevlake/subtasks.py new file mode 100644 index 00000000000..98591d331e6 --- /dev/null +++ b/backend/python/pydevlake/pydevlake/subtasks.py @@ -0,0 +1,215 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from abc import abstractmethod +import json +from datetime import datetime +from typing import Tuple, Dict, Iterable, Optional, Generator + + +import sqlalchemy.sql as sql +from sqlmodel import Session, SQLModel, Field, select + +from pydevlake.model import RawModel, ToolModel, DomainModel, generate_domain_id +from pydevlake.context import Context +from pydevlake.message import RemoteProgress +from pydevlake import logger + + +class Subtask: + def __init__(self, stream): + self.stream = stream + + @property + def name(self): + return f'{self.verb.lower()}{self.stream.plugin_name.capitalize()}{self.stream.name.capitalize()}' + + @property + def description(self): + return f'{self.verb.capitalize()} {self.stream.plugin_name} {self.stream.name.lower()}' + + @property + def verb(self) -> str: + pass + + def run(self, ctx: Context, sync_point_interval=100): + with Session(ctx.engine) as session: + subtask_run = self._start_subtask(session, ctx.connection_id) + if ctx.incremental: + state = self._get_last_state(session, ctx.connection_id) + else: + self.delete(session, ctx) + state = dict() + + try: + for i, (data, state) in enumerate(self.fetch(state, session, ctx)): + self.process(data, session, ctx) + + if i % sync_point_interval == 0 and i != 0: + # Save current state + subtask_run.state = json.dumps(state) + session.merge(subtask_run) + session.commit() + + # Send progress + yield RemoteProgress( + increment=sync_point_interval, + current=i + ) + except Exception as e: + logger.error(e) + + subtask_run.state = json.dumps(state) + subtask_run.completed = datetime.now() + session.merge(subtask_run) + session.commit() + + def _start_subtask(self, session, connection_id): + subtask_run = SubtaskRun( + subtask_name=self.name, + connection_id=connection_id, + started=datetime.now(), + state=json.dumps({}) + ) + session.add(subtask_run) + return subtask_run + + @abstractmethod + def fetch(self, state: Dict, session: Session, ctx: Context) -> Iterable[Tuple[object, Dict]]: + """ + Queries the data source and returns an iterable of (data, state) tuples. + The `data` can be any object. + The `state` is a dict with str keys. + `Fetch` is called with the last state of the last run of this subtask. + """ + pass + + @abstractmethod + def process(self, data: object, session: Session): + """ + Called for all data entries returned by `fetch`. + """ + pass + + def _get_last_state(self, session, connection_id): + stmt = ( + select(SubtaskRun) + .where(SubtaskRun.subtask_name == self.name) + .where(SubtaskRun.connection_id == connection_id) + .where(SubtaskRun.completed != None) + .order_by(SubtaskRun.started) + ) + subtask_run = session.exec(stmt).first() + if subtask_run is not None: + return json.loads(subtask_run.state) + return {} + + +class SubtaskRun(SQLModel, table=True): + """ + Table storing information about the execution of subtasks. + + #TODO: rework id uniqueness: + # sync with Keon about the table he created for Singer MR + """ + id: Optional[int] = Field(primary_key=True) + subtask_name: str + connection_id: int + started: datetime + completed: Optional[datetime] + state: str # JSON encoded dict of atomic values + + +class Collector(Subtask): + @property + def verb(self): + return 'collect' + + def fetch(self, state: Dict, _, ctx: Context) -> Iterable[Tuple[object, Dict]]: + return self.stream.collect(state, ctx) + + def process(self, data: object, session: Session, ctx: Context): + raw_model_class = self.stream.raw_model(session) + raw_model = raw_model_class( + params=self._params(ctx), + data=json.dumps(data).encode('utf8') + ) + session.add(raw_model) + + def _params(self, ctx: Context) -> str: + return json.dumps({ + "connection_id": ctx.connection_id, + "scope_id": ctx.options['scopeId'] + }) + + def delete(self, session, ctx): + raw_model = self.stream.raw_model(session) + stmt = sql.delete(raw_model).where(raw_model.params == self._params(ctx)) + session.execute(stmt) + + +class SubstreamCollector(Collector): + def fetch(self, state: Dict, session, ctx: Context): + for parent in session.exec(sql.select(self.stream.parent_stream.tool_model)).scalars(): + yield from self.stream.collect(state, ctx, parent) + + +class Extractor(Subtask): + @property + def verb(self): + return 'extract' + + def fetch(self, state: Dict, session: Session, ctx: Context) -> Iterable[Tuple[object, dict]]: + raw_model = self.stream.raw_model(session) + # TODO: Should filter for same options? + for raw in session.query(raw_model).all(): + yield raw, state + + def process(self, raw: RawModel, session: Session, _): + tool_model = self.stream.extract(json.loads(raw.data)) + tool_model.set_origin(raw) + session.merge(tool_model) + + def delete(self, session, ctx): + pass + +class Convertor(Subtask): + @property + def verb(self): + return 'convert' + + def fetch(self, state: Dict, session: Session, _) -> Iterable[Tuple[ToolModel, Dict]]: + for item in session.query(self.stream.tool_model).all(): + yield item, state + + def process(self, tool_model: ToolModel, session: Session, ctx: Context): + res = self.stream.convert(tool_model) + if isinstance(res, Generator): + for each in self.stream.convert(tool_model): + self._save(tool_model, each, session, ctx.connection_id) + else: + self._save(tool_model, res, session, ctx.connection_id) + + def _save(self, tool_model: ToolModel, domain_model: DomainModel, session: Session, connection_id: int): + if not isinstance(domain_model, DomainModel): + logger.error(f'Expected a DomainModel but got a {type(domain_model)}: {domain_model}') + return + + domain_model.id = generate_domain_id(tool_model, connection_id) + session.merge(domain_model) + + def delete(self, session, ctx): + pass diff --git a/backend/python/pydevlake/pyproject.toml b/backend/python/pydevlake/pyproject.toml new file mode 100644 index 00000000000..e46ee96e55b --- /dev/null +++ b/backend/python/pydevlake/pyproject.toml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[tool.poetry] +name = "pydevlake" +version = "0.1.0" +description = "Devlake plugin framework" +authors = ["Camille Teruel ", "Keon Amini "] +license = "Apache-2.0" +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.10" +sqlmodel = "^0.0.8" +mysqlclient = "^2.1.1" +requests = "^2.28.1" +inflect = "^6.0.2" +fire = "^0.4.0" +pydantic = "^1.10.2" +pydevd-pycharm = "^231.6471.3" + + +[tool.poetry.group.dev.dependencies] +pytest = "^7.2.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/backend/python/pydevlake/test/remote_test.go b/backend/python/pydevlake/test/remote_test.go new file mode 100644 index 00000000000..9aaeec87737 --- /dev/null +++ b/backend/python/pydevlake/test/remote_test.go @@ -0,0 +1,100 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/server/services/remote" + "github.com/apache/incubator-devlake/server/services/remote/bridge" + "github.com/apache/incubator-devlake/server/services/remote/models" + plg "github.com/apache/incubator-devlake/server/services/remote/plugin" +) + +type CircleCIConnection struct { + ID uint64 `json:"id"` + Token string `json:"token" encrypt:"yes"` +} + +func CreateRemotePlugin(t *testing.T) models.RemotePlugin { + // TODO: Create a dummy plugin for tests instead of using CircleCI plugin + pluginCmdPath := "../../plugins/circle_ci/circle_ci/main.py" + invoker := bridge.NewPythonPoetryCmdInvoker(pluginCmdPath) + + pluginInfo := models.PluginInfo{} + err := invoker.Call("plugin-info", bridge.DefaultContext).Get(&pluginInfo) + + if err != nil { + t.Error("Cannot get plugin info", err) + return nil + } + + remotePlugin, err := remote.NewRemotePlugin(&pluginInfo) + if err != nil { + t.Error("Cannot create remote plugin", err) + return nil + } + + return remotePlugin +} + +func TestCreateRemotePlugin(t *testing.T) { + _ = CreateRemotePlugin(t) +} + +func TestRunSubTask(t *testing.T) { + remotePlugin := CreateRemotePlugin(t) + dataflowTester := e2ehelper.NewDataFlowTester(t, "circleci", remotePlugin) + subtask := remotePlugin.SubTaskMetas()[0] + options := make(map[string]interface{}) + options["project_slug"] = "gh/circleci/bond" + options["scopeId"] = "1" + taskData := plg.RemotePluginTaskData{ + DbUrl: bridge.DefaultContext.GetConfig("db_url"), + Connection: CircleCIConnection{ID: 1}, + Options: options, + } + dataflowTester.Subtask(subtask, taskData) +} + +func TestTestConnection(t *testing.T) { + remotePlugin := CreateRemotePlugin(t) + + var handler plugin.ApiResourceHandler + for resource, endpoints := range remotePlugin.ApiResources() { + if resource == "test" { + handler = endpoints["POST"] + } + } + + if handler == nil { + t.Error("Missing test connection API resource") + } + + input := plugin.ApiResourceInput{ + Body: map[string]interface{}{ + "token": "secret", + }, + } + _, err := handler(&input) + if err != nil { + t.Error(err) + } +} diff --git a/backend/python/pydevlake/test/stream_test.py b/backend/python/pydevlake/test/stream_test.py new file mode 100644 index 00000000000..db24465a275 --- /dev/null +++ b/backend/python/pydevlake/test/stream_test.py @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json + +import pytest +from sqlmodel import Session, Field + +from pydevlake import Stream, Connection, Context +from pydevlake.model import ToolModel, DomainModel + + +class DummyToolModel(ToolModel, table=True): + id: int = Field(primary_key=True) + name: str + + +class DummyDomainModel(DomainModel, table=True): + Name: str + + +class DummyStream(Stream): + tool_model=DummyToolModel + domain_model=DummyDomainModel + + def collect(self, state, context): + for i, each in enumerate(context.connection.raw_data): + count = state.get("count", 0) + yield each, {"count": count+i} + + def extract(self, raw) -> ToolModel: + return DummyToolModel( + id=raw["i"], + name=raw["n"] + ) + + def convert(self, tm): + return DummyDomainModel( + ID=tm.id, + Name=tm.name, + ) + + +class DummyConnection(Connection): + raw_data: list[dict] + + +@pytest.fixture +def raw_data(): + return [ + {"i": 1, "n": "alice"}, + {"i": 2, "n": "bob"} + ] + + +@pytest.fixture +def connection(raw_data): + return DummyConnection(raw_data=raw_data) + + +@pytest.fixture +def ctx(connection): + return Context( + db_url="sqlite+pysqlite:///:memory:", + connection_id=11, + connection=connection, + options={"scopeId": 1, "scopeName": "foo"} + ) + + +@pytest.fixture +def stream(): + return DummyStream("test") + +def test_collect_data(stream, raw_data, ctx): + gen = stream.collector.run(ctx) + list(gen) + + with Session(ctx.engine) as session: + raw_model = stream.raw_model(session) + all_raw = [json.loads(r.data) for r in session.query(raw_model).all()] + assert all_raw == raw_data + + +def test_extract_data(stream, raw_data, ctx): + with Session(ctx.engine) as session: + for each in raw_data: + raw_model = stream.raw_model(session) + session.add(raw_model(data=json.dumps(each))) + session.commit() + + gen = stream.extractor.run(ctx) + list(gen) + + tool_models = session.query(DummyToolModel).all() + alice = tool_models[0] + bob = tool_models[1] + assert alice.name == 'alice' + assert alice.id == 1 + + assert bob.name == 'bob' + assert bob.id == 2 + + +def test_convert_data(stream, raw_data, ctx): + with Session(ctx.engine) as session: + for each in raw_data: + session.add( + DummyToolModel( + id=each["i"], + name=each["n"], + raw_data_table="_raw_dummy_model", + raw_data_params=json.dumps({"connection_id": ctx.connection_id, "scope_id": ctx.options.scopeId}) + ) + ) + session.commit() + + gen = stream.convertor.run(ctx) + list(gen) + + tool_models = session.query(DummyDomainModel).all() + alice = tool_models[0] + bob = tool_models[1] + assert alice.Name == 'alice' + assert alice.id == 'test:DummyToolModel:11:1' + + assert bob.Name == 'bob' + assert bob.id == 'test:DummyToolModel:11:2' diff --git a/backend/python/requirements.txt b/backend/python/requirements.txt index a33f6c92871..6575706b48e 100644 --- a/backend/python/requirements.txt +++ b/backend/python/requirements.txt @@ -1,3 +1,3 @@ +# runtime miscellaneous dependencies setuptools -wheel -tap-pagerduty==0.2.0 \ No newline at end of file +wheel \ No newline at end of file diff --git a/backend/server/api/api.go b/backend/server/api/api.go index 71a94c7deb3..46a2802ed91 100644 --- a/backend/server/api/api.go +++ b/backend/server/api/api.go @@ -18,18 +18,23 @@ limitations under the License. package api import ( + "net/http" + "strconv" + "strings" + "time" + "github.com/apache/incubator-devlake/core/config" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/impls/logruslog" + _ "github.com/apache/incubator-devlake/server/api/docs" + "github.com/apache/incubator-devlake/server/api/remote" "github.com/apache/incubator-devlake/server/api/shared" "github.com/apache/incubator-devlake/server/services" - "net/http" - "time" - - _ "github.com/apache/incubator-devlake/server/api/docs" + "github.com/apache/incubator-devlake/server/services/remote/bridge" "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" + "github.com/spf13/viper" ginSwagger "github.com/swaggo/gin-swagger" "github.com/swaggo/gin-swagger/swaggerFiles" ) @@ -52,7 +57,10 @@ func CreateApiService() { v := config.GetConfig() gin.SetMode(v.GetString("MODE")) router := gin.Default() - + remotePluginsEnabled := v.GetBool("ENABLE_REMOTE_PLUGINS") + if remotePluginsEnabled { + router.POST("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/plugins/register", remote.RegisterPlugin(router, registerPluginEndpoints)) + } // Wait for user confirmation if db migration is needed router.GET("/proceed-db-migration", func(ctx *gin.Context) { if !services.MigrationRequireConfirmation() { @@ -95,8 +103,24 @@ func CreateApiService() { })) RegisterRouter(router) - err := router.Run(v.GetString("PORT")) + port := v.GetString("PORT") + if remotePluginsEnabled { + go bootstrapRemotePlugins(v) + } + err := router.Run(port) if err != nil { panic(err) } } + +func bootstrapRemotePlugins(v *viper.Viper) { + port := v.GetString("PORT") + portNum, err := strconv.Atoi(strings.Split(port, ":")[1]) + if err != nil { + panic(err) + } + err = bridge.Bootstrap(v, portNum) + if err != nil { + logruslog.Global.Error(err, "") + } +} diff --git a/backend/server/api/remote/models.go b/backend/server/api/remote/models.go new file mode 100644 index 00000000000..079f27cdf86 --- /dev/null +++ b/backend/server/api/remote/models.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package remote + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/server/services/remote/models" +) + +type SwaggerDoc struct { + Name string `json:"name" validate:"required"` + Resource string `json:"resource" validate:"required"` + Spec json.RawMessage `json:"spec" validate:"required"` +} + +type PluginDetails struct { + PluginInfo models.PluginInfo `json:"plugin_info" validate:"required"` + Swagger SwaggerDoc `json:"swagger" validate:"required"` +} diff --git a/backend/server/api/remote/register.go b/backend/server/api/remote/register.go new file mode 100644 index 00000000000..1c886027b4b --- /dev/null +++ b/backend/server/api/remote/register.go @@ -0,0 +1,107 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package remote + +import ( + "fmt" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/server/api/shared" + "github.com/apache/incubator-devlake/server/services/remote" + + "github.com/RaveNoX/go-jsonmerge" + "github.com/gin-gonic/gin" + "github.com/go-playground/validator/v10" + ginSwagger "github.com/swaggo/gin-swagger" + "github.com/swaggo/gin-swagger/swaggerFiles" + "github.com/swaggo/swag" +) + +var ( + vld = validator.New() + cachedDocs = map[string]*swag.Spec{} +) + +type ApiResource struct { + PluginName string + Resources map[string]map[string]plugin.ApiResourceHandler +} + +// TODO add swagger doc +func RegisterPlugin(router *gin.Engine, registerEndpoints func(r *gin.Engine, pluginName string, apiResources map[string]map[string]plugin.ApiResourceHandler)) func(*gin.Context) { + return func(c *gin.Context) { + var details PluginDetails + if err := c.ShouldBindJSON(&details); err != nil { + shared.ApiOutputError(c, errors.BadInput.Wrap(err, shared.BadRequestBody)) + return + } + if err := vld.Struct(&details); err != nil { + shared.ApiOutputError(c, errors.BadInput.Wrap(err, shared.BadRequestBody)) + return + } + remotePlugin, err := remote.NewRemotePlugin(&details.PluginInfo) + if err != nil { + shared.ApiOutputError(c, errors.Default.Wrap(err, "plugin could not be initialized")) + return + } + resource := ApiResource{ + PluginName: details.PluginInfo.Name, + Resources: remotePlugin.ApiResources(), + } + registerEndpoints(router, resource.PluginName, resource.Resources) + registerSwagger(router, &details.Swagger) + shared.ApiOutputSuccess(c, nil, http.StatusOK) + } +} + +func registerSwagger(router *gin.Engine, doc *SwaggerDoc) { + if spec, ok := cachedDocs[doc.Name]; ok { + spec.SwaggerTemplate = combineSpecs(spec.SwaggerTemplate, string(doc.Spec)) + } else { + spec = &swag.Spec{ + Version: "", + Host: "", + BasePath: "", + Schemes: nil, + Title: "", + Description: "", + InfoInstanceName: doc.Name, + SwaggerTemplate: string(doc.Spec), + } + swag.Register(doc.Name, spec) + cachedDocs[doc.Name] = spec + router.GET(fmt.Sprintf("/plugins/swagger/%s/*any", doc.Resource), ginSwagger.CustomWrapHandler( + &ginSwagger.Config{ + URL: "doc.json", + DocExpansion: "list", + InstanceName: doc.Name, + Title: "", + DefaultModelsExpandDepth: 1, + DeepLinking: true, + PersistAuthorization: false, + }, + swaggerFiles.Handler)) + } +} + +func combineSpecs(spec1 string, spec2 string) string { + res, _ := jsonmerge.Merge(spec1, spec2) + return res.(string) +} diff --git a/backend/server/api/router.go b/backend/server/api/router.go index 41bf887a8a6..8408a7af073 100644 --- a/backend/server/api/router.go +++ b/backend/server/api/router.go @@ -19,6 +19,9 @@ package api import ( "fmt" + "net/http" + "strings" + "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/server/api/blueprints" "github.com/apache/incubator-devlake/server/api/domainlayer" @@ -31,8 +34,6 @@ import ( "github.com/apache/incubator-devlake/server/api/task" "github.com/apache/incubator-devlake/server/api/version" "github.com/apache/incubator-devlake/server/services" - "net/http" - "strings" "github.com/gin-gonic/gin" ) @@ -73,24 +74,29 @@ func RegisterRouter(r *gin.Engine) { r.GET("/projects", project.GetProjects) // mount all api resources for all plugins - pluginsApiResources, err := services.GetPluginsApiResources() + resources, err := services.GetPluginsApiResources() if err != nil { panic(err) } - for pluginName, apiResources := range pluginsApiResources { - for resourcePath, resourceHandlers := range apiResources { - for method, h := range resourceHandlers { - r.Handle( - method, - fmt.Sprintf("/plugins/%s/%s", pluginName, resourcePath), - handlePluginCall(pluginName, h), - ) - } + // mount all api resources for all plugins + for pluginName, apiResources := range resources { + registerPluginEndpoints(r, pluginName, apiResources) + } +} + +func registerPluginEndpoints(r *gin.Engine, pluginName string, apiResources map[string]map[string]plugin.ApiResourceHandler) { + for resourcePath, resourceHandlers := range apiResources { + for method, h := range resourceHandlers { + r.Handle( + method, + fmt.Sprintf("/plugins/%s/%s", pluginName, resourcePath), + handlePluginCall(h), + ) } } } -func handlePluginCall(pluginName string, handler plugin.ApiResourceHandler) func(c *gin.Context) { +func handlePluginCall(handler plugin.ApiResourceHandler) func(c *gin.Context) { return func(c *gin.Context) { var err error input := &plugin.ApiResourceInput{} diff --git a/backend/server/services/remote/bridge/bootstrap.go b/backend/server/services/remote/bridge/bootstrap.go new file mode 100644 index 00000000000..f686243f9e5 --- /dev/null +++ b/backend/server/services/remote/bridge/bootstrap.go @@ -0,0 +1,59 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bridge + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + "github.com/apache/incubator-devlake/impls/logruslog" + "github.com/spf13/viper" +) + +func Bootstrap(cfg *viper.Viper, port int) errors.Error { + scriptPath := cfg.GetString("REMOTE_PLUGINS_STARTUP_PATH") + if scriptPath == "" { + return errors.BadInput.New(fmt.Sprintf("missing env key: %s", "REMOTE_PLUGINS_STARTUP_PATH")) + } + workingDir, err := errors.Convert01(os.Getwd()) + if err != nil { + return err + } + absScriptPath := filepath.Join(workingDir, scriptPath) + cmd := exec.Command(absScriptPath, fmt.Sprintf("http://127.0.0.1:%d", port)) //expects the plugins to live on the same host + cmd.Dir = filepath.Dir(absScriptPath) + result, err := utils.RunProcess(cmd, &utils.RunProcessOptions{ + OnStdout: func(b []byte) { + logruslog.Global.Info(string(b)) + }, + OnStderr: func(b []byte) { + logruslog.Global.Error(nil, string(b)) + }, + }) + if err != nil { + return err + } + if result.GetError() != nil { + logruslog.Global.Error(result.GetError(), "error occurred bootstrapping remote plugins") + } + return nil +} diff --git a/backend/server/services/remote/bridge/bridge.go b/backend/server/services/remote/bridge/bridge.go new file mode 100644 index 00000000000..2d80685ddca --- /dev/null +++ b/backend/server/services/remote/bridge/bridge.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bridge + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/server/services/remote/models" +) + +type ( + Bridge struct { + invoker Invoker + } + Invoker interface { + Call(methodName string, ctx plugin.ExecContext, args ...any) *CallResult + Stream(methodName string, ctx plugin.ExecContext, args ...any) *MethodStream + } +) + +func NewBridge(invoker Invoker) *Bridge { + return &Bridge{invoker: invoker} +} + +func (b *Bridge) RemoteSubtaskEntrypointHandler(subtaskMeta models.SubtaskMeta) plugin.SubTaskEntryPoint { + return func(ctx plugin.SubTaskContext) errors.Error { + args := []interface{}{ctx.GetData()} + for _, arg := range subtaskMeta.Arguments { + args = append(args, arg) + } + stream := b.invoker.Stream(subtaskMeta.EntryPointName, NewChildRemoteContext(ctx), args...) + for recv := range stream.Receive() { + if recv.err != nil { + return recv.err + } + progress := RemoteProgress{} + err := recv.Get(&progress) + if err != nil { + return err + } + if progress.Current != 0 { + ctx.SetProgress(progress.Current, progress.Total) + } else if progress.Increment != 0 { + ctx.IncProgress(progress.Increment) + } + } + return nil + } +} diff --git a/backend/server/services/remote/bridge/cmd.go b/backend/server/services/remote/bridge/cmd.go new file mode 100644 index 00000000000..b3a14371aef --- /dev/null +++ b/backend/server/services/remote/bridge/cmd.go @@ -0,0 +1,156 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bridge + +import ( + "fmt" + "os/exec" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" +) + +type CmdInvoker struct { + resolveCmd func(methodName string, args ...string) (string, []string) + cancelled bool + workingPath string +} + +func NewCmdInvoker(workingPath string, resolveCmd func(methodName string, args ...string) (string, []string)) *CmdInvoker { + return &CmdInvoker{ + resolveCmd: resolveCmd, + workingPath: workingPath, + } +} + +func (c *CmdInvoker) Call(methodName string, ctx plugin.ExecContext, args ...any) *CallResult { + serializedArgs, err := serialize(args...) + if err != nil { + return &CallResult{ + err: err, + } + } + executable, inputArgs := c.resolveCmd(methodName, serializedArgs...) + cmdCtx := DefaultContext.GetContext() + cmd := exec.CommandContext(cmdCtx, executable, inputArgs...) + if c.workingPath != "" { + cmd.Dir = c.workingPath + } + response, err := utils.RunProcess(cmd, &utils.RunProcessOptions{ + OnStdout: func(b []byte) { + msg := string(b) + c.logRemoteMessage(ctx.GetLogger(), msg) + }, + OnStderr: func(b []byte) { + msg := string(b) + c.logRemoteError(ctx.GetLogger(), msg) + }, + UseFdOut: true, + }) + if err != nil { + return NewCallResult(nil, err) + } + err = response.GetError() + if err != nil { + return &CallResult{ + err: errors.Default.Wrap(err, fmt.Sprintf("failed to invoke remote function \"%s\"", methodName)), + } + } + results, err := deserialize(response.GetFdOut().([]byte)) + return NewCallResult(results, errors.Convert(err)) +} + +func (c *CmdInvoker) Stream(methodName string, ctx plugin.ExecContext, args ...any) *MethodStream { + recvChannel := make(chan *StreamResult) + stream := &MethodStream{ + outbound: nil, + inbound: recvChannel, + } + serializedArgs, err := serialize(args...) + if err != nil { + recvChannel <- NewStreamResult(nil, err) + return stream + } + executable, inputArgs := c.resolveCmd(methodName, serializedArgs...) + cmdCtx := DefaultContext.GetContext() // grabbing context off of ctx kills the cmd after a couple of seconds... why? + cmd := exec.CommandContext(cmdCtx, executable, inputArgs...) + if c.workingPath != "" { + cmd.Dir = c.workingPath + } + processHandle, err := utils.StreamProcess(cmd, &utils.StreamProcessOptions{ + OnStdout: func(b []byte) (any, errors.Error) { + msg := string(b) + c.logRemoteMessage(ctx.GetLogger(), msg) + return b, nil + }, + OnStderr: func(b []byte) (any, errors.Error) { + msg := string(b) + c.logRemoteError(ctx.GetLogger(), msg) + return b, nil + }, + UseFdOut: true, + OnFdOut: utils.NoopConverter, + }) + if err != nil { + recvChannel <- NewStreamResult(nil, err) + return stream + } + go func() { + defer close(recvChannel) + for msg := range processHandle.Receive() { + if err = msg.GetError(); err != nil { + recvChannel <- NewStreamResult(nil, err) + } + if !c.cancelled { + select { + case <-ctx.GetContext().Done(): + err = processHandle.Cancel() + if err != nil { + recvChannel <- NewStreamResult(nil, errors.Default.Wrap(err, "error cancelling python target")) + return + } + c.cancelled = true + // continue until the stream gets closed by the child + default: + } + } + response := msg.GetFdOut() + if response != nil { + results, err := deserialize(response.([]byte)) + if err != nil { + recvChannel <- NewStreamResult(nil, err) + } else { + recvChannel <- NewStreamResult(results, nil) + } + } + } + }() + return stream +} + +func (c *CmdInvoker) logRemoteMessage(logger log.Logger, msg string) { + logger.Info(msg) +} + +func (c *CmdInvoker) logRemoteError(logger log.Logger, msg string) { + logger.Error(nil, msg) +} + +var _ Invoker = (*CmdInvoker)(nil) diff --git a/backend/server/services/remote/bridge/context.go b/backend/server/services/remote/bridge/context.go new file mode 100644 index 00000000000..57dd78dae95 --- /dev/null +++ b/backend/server/services/remote/bridge/context.go @@ -0,0 +1,140 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bridge + +import ( + "context" + + "github.com/apache/incubator-devlake/core/config" + ctx "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/impls/logruslog" + "github.com/spf13/cast" + "github.com/spf13/viper" +) + +var DefaultContext = NewRemoteContext(logruslog.Global, config.GetConfig()) + +type RemoteProgress struct { + Current int `json:"current"` + Total int `json:"total"` + Increment int `json:"increment"` +} + +type RemoteContext interface { + plugin.ExecContext + GetSettings() map[string]any +} + +type remoteContextImpl struct { + parent plugin.ExecContext + logger log.Logger + ctx context.Context + Settings map[string]any `json:"settings"` +} + +func (r remoteContextImpl) GetSettings() map[string]any { + return r.Settings +} + +func (r remoteContextImpl) GetConfigReader() config.ConfigReader { + return r.parent.GetConfigReader() +} + +func (r remoteContextImpl) ReplaceLogger(logger log.Logger) ctx.BasicRes { + return &remoteContextImpl{ + parent: r.parent, + logger: logger, + ctx: r.ctx, + Settings: r.Settings, + } +} + +func (r remoteContextImpl) NestedLogger(name string) ctx.BasicRes { + return r.ReplaceLogger(r.logger.Nested(name)) +} + +func NewRemoteContext(logger log.Logger, cfg *viper.Viper) RemoteContext { + return &remoteContextImpl{ + logger: logger, + Settings: cfg.AllSettings(), + ctx: context.Background(), + } +} + +func NewChildRemoteContext(ec plugin.ExecContext) RemoteContext { + return &remoteContextImpl{ + parent: ec, + logger: ec.GetLogger(), + ctx: ec.GetContext(), + Settings: DefaultContext.GetSettings(), + } +} + +func (r remoteContextImpl) GetConfig(name string) string { + val, ok := r.Settings[name] + if !ok { + return "" + } + return cast.ToString(val) +} + +func (r remoteContextImpl) GetLogger() log.Logger { + return r.logger +} + +func (r remoteContextImpl) GetDal() dal.Dal { + if r.parent != nil { + return r.parent.GetDal() + } + return nil +} + +func (r remoteContextImpl) GetName() string { + if r.parent != nil { + return r.parent.GetName() + } + return "default_remote" +} + +func (r remoteContextImpl) GetContext() context.Context { + return r.ctx +} + +func (r remoteContextImpl) GetData() interface{} { + if r.parent != nil { + return r.parent.GetData() + } + return nil +} + +func (r remoteContextImpl) SetProgress(current int, total int) { + if r.parent != nil { + r.parent.SetProgress(current, total) + } +} + +func (r remoteContextImpl) IncProgress(quantity int) { + if r.parent != nil { + r.parent.IncProgress(quantity) + } +} + +var _ RemoteContext = (*remoteContextImpl)(nil) diff --git a/backend/server/services/remote/bridge/python_cmd.go b/backend/server/services/remote/bridge/python_cmd.go new file mode 100644 index 00000000000..bdcebcd82a9 --- /dev/null +++ b/backend/server/services/remote/bridge/python_cmd.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bridge + +import ( + "path/filepath" + "strings" +) + +const ( + poetryExec = "poetry" + pythonExec = "python" +) + +func NewPythonPoetryCmdInvoker(scriptPath string) *CmdInvoker { + tomlPath := filepath.Dir(filepath.Dir(scriptPath)) //the main entrypoint expected to be at toplevel + scriptPath = strings.TrimPrefix(scriptPath, tomlPath+"/") + return NewCmdInvoker(tomlPath, func(methodName string, args ...string) (string, []string) { + allArgs := []string{"run", pythonExec, scriptPath, methodName} + allArgs = append(allArgs, args...) + return poetryExec, allArgs + }) +} diff --git a/backend/server/services/remote/bridge/returns.go b/backend/server/services/remote/bridge/returns.go new file mode 100644 index 00000000000..b7aa46fb65f --- /dev/null +++ b/backend/server/services/remote/bridge/returns.go @@ -0,0 +1,88 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bridge + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/mitchellh/mapstructure" +) + +type ( + CallResult struct { + results []map[string]any + err errors.Error + } + StreamResult = CallResult + MethodStream struct { + // send (not supported for now) + outbound chan<- any + // receive + inbound <-chan *StreamResult + } +) + +func NewCallResult(results []map[string]any, err errors.Error) *CallResult { + return &CallResult{ + results: results, + err: err, + } +} + +func (m *CallResult) Get(targets ...any) errors.Error { + if m.err != nil { + return m.err + } + if len(targets) != len(m.results) { + // if everything came back as nil, consider it good + for _, result := range m.results { + if result != nil { + return errors.Default.New("unequal results and targets length") + } + } + return nil + } + + for i, target := range targets { + config := &mapstructure.DecoderConfig{ + TagName: "json", + Result: target, + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return errors.Convert(err) + } + + err = decoder.Decode(m.results[i]) + if err != nil { + return errors.Convert(err) + } + } + return nil +} + +func NewStreamResult(results []map[string]any, err errors.Error) *StreamResult { + return &StreamResult{ + results: results, + err: err, + } +} + +func (m *MethodStream) Receive() <-chan *StreamResult { + return m.inbound +} diff --git a/backend/server/services/remote/bridge/utils.go b/backend/server/services/remote/bridge/utils.go new file mode 100644 index 00000000000..542074e8785 --- /dev/null +++ b/backend/server/services/remote/bridge/utils.go @@ -0,0 +1,57 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bridge + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" +) + +func serialize(args ...any) ([]string, errors.Error) { + var serializedArgs []string + for _, arg := range args { + serializedArg, err := json.Marshal(arg) + if err != nil { + return nil, errors.Convert(err) + } + serializedArgs = append(serializedArgs, string(serializedArg)) + } + return serializedArgs, nil +} + +func deserialize(bytes json.RawMessage) ([]map[string]any, errors.Error) { + if len(bytes) == 0 { + return nil, nil + } + var result []map[string]any + if bytes[0] == '{' { + single := make(map[string]any) + if err := json.Unmarshal(bytes, &single); err != nil { + return nil, errors.Convert(err) + } + result = append(result, single) + } else if bytes[0] == '[' { + if err := json.Unmarshal(bytes, &result); err != nil { + return nil, errors.Convert(err) + } + } else { + return nil, errors.Default.New("malformed JSON from remote call") + } + return result, nil +} diff --git a/backend/server/services/remote/init.go b/backend/server/services/remote/init.go new file mode 100644 index 00000000000..98d46770dfb --- /dev/null +++ b/backend/server/services/remote/init.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package remote + +import ( + "github.com/apache/incubator-devlake/core/config" + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + pluginCore "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/server/services/remote/models" + remote "github.com/apache/incubator-devlake/server/services/remote/plugin" +) + +var ( + remotePlugins = make(map[string]models.RemotePlugin) +) + +func Init(br context.BasicRes) { + remote.Init(br) +} + +func NewRemotePlugin(info *models.PluginInfo) (models.RemotePlugin, errors.Error) { + if _, ok := remotePlugins[info.Name]; ok { + return nil, errors.BadInput.New("plugin already registered") + } + plugin, err := remote.NewRemotePlugin(info) + if err != nil { + return nil, errors.BadInput.New("unsupported plugin type") + } + forceMigration := config.GetConfig().GetBool("FORCE_MIGRATION") + err = plugin.RunMigrations(forceMigration) + if err != nil { + return nil, err + } + err = pluginCore.RegisterPlugin(info.Name, plugin) + if err != nil { + return nil, err + } + remotePlugins[info.Name] = plugin + return plugin, nil +} diff --git a/backend/server/services/remote/models/conversion.go b/backend/server/services/remote/models/conversion.go new file mode 100644 index 00000000000..5b07efdaed1 --- /dev/null +++ b/backend/server/services/remote/models/conversion.go @@ -0,0 +1,100 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "fmt" + "reflect" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/common" +) + +func LoadTableModel(tableName string, schema map[string]any) (*models.DynamicTabler, errors.Error) { + structType, err := GenerateStructType(schema, reflect.TypeOf(common.Model{})) + if err != nil { + return nil, err + } + return models.NewDynamicTabler(tableName, structType), nil +} + +func GenerateStructType(schema map[string]any, baseType reflect.Type) (reflect.Type, errors.Error) { + var structFields []reflect.StructField + propsRaw, ok := schema["properties"] + if !ok { + return nil, errors.BadInput.New("Missing properties in JSON schema") + } + props, ok := propsRaw.(map[string]any) + if !ok { + return nil, errors.BadInput.New("JSON schema properties must be an object") + } + if baseType != nil { + anonymousField := reflect.StructField{ + Name: baseType.Name(), + Type: baseType, + Tag: reflect.StructTag("mapstructure:\",squash\""), + Anonymous: true, + } + structFields = append(structFields, anonymousField) + } + for k, v := range props { + spec := v.(map[string]any) + field, err := generateStructField(k, spec) + if err != nil { + return nil, err + } + structFields = append(structFields, *field) + } + return reflect.StructOf(structFields), nil +} + +func generateStructField(name string, schema map[string]any) (*reflect.StructField, errors.Error) { + tag := reflect.StructTag(fmt.Sprintf("json:\"%s\" "+ + "gorm:\"serializer:encdec\"", //just encrypt everything for GORM operations - makes things easy + name)) + goType, err := getGoType(schema) + if err != nil { + return nil, err + } + return &reflect.StructField{ + Name: strings.Title(name), //nolint:staticcheck + Type: goType, + Tag: tag, + }, nil +} + +func getGoType(schema map[string]any) (reflect.Type, errors.Error) { + var goType reflect.Type + jsonType, ok := schema["type"].(string) + if !ok { + return nil, errors.BadInput.New("\"type\" property must be a string") + } + switch jsonType { + //TODO: support more types + case "integer": + goType = reflect.TypeOf(uint64(0)) + case "string": + //TODO: distinguish stypes based on string format + goType = reflect.TypeOf("") + default: + return nil, errors.BadInput.New(fmt.Sprintf("Unsupported type %s", jsonType)) + } + return goType, nil +} diff --git a/backend/server/services/remote/models/models.go b/backend/server/services/remote/models/models.go new file mode 100644 index 00000000000..d19a26fe792 --- /dev/null +++ b/backend/server/services/remote/models/models.go @@ -0,0 +1,58 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +const ( + PythonPoetryCmd PluginType = "python-poetry" + PythonCmd PluginType = "python" + None PluginExtension = "" + Metric PluginExtension = "metric" + Datasource PluginExtension = "datasource" +) + +type ( + PluginType string + PluginExtension string +) + +type PluginInfo struct { + Type PluginType `json:"type" validate:"required"` + Name string `json:"name" validate:"required"` + Extension PluginExtension `json:"extension"` + ConnectionSchema map[string]any `json:"connection_schema" validate:"required"` + Description string `json:"description"` + PluginPath string `json:"plugin_path" validate:"required"` + ApiEndpoints []Endpoint `json:"api_endpoints" validate:"dive"` + SubtaskMetas []SubtaskMeta `json:"subtask_metas" validate:"dive"` +} + +type SubtaskMeta struct { + Name string `json:"name" validate:"required"` + EntryPointName string `json:"entry_point_name" validate:"required"` + Arguments []string `json:"arguments"` + Required bool `json:"required"` + EnabledByDefault bool `json:"enabled_by_default"` + Description string `json:"description" validate:"required"` + DomainTypes []string `json:"domain_types" validate:"required"` +} + +type Endpoint struct { + Resource string `json:"resource" validate:"required"` + Handler string `json:"handler" validate:"required"` + Method string `json:"method" validate:"required"` +} diff --git a/backend/server/services/remote/models/plugin_models.go b/backend/server/services/remote/models/plugin_models.go new file mode 100644 index 00000000000..4284112d6e1 --- /dev/null +++ b/backend/server/services/remote/models/plugin_models.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +type ( + PipelineScope struct { + ScopeId string + ScopeName string + TableName string + } + WrappedPipelineScope struct { + Scope PipelineScope + } +) + +func (p *WrappedPipelineScope) ScopeId() string { + return p.Scope.ScopeId +} + +func (p *WrappedPipelineScope) ScopeName() string { + return p.Scope.ScopeName +} + +func (p *WrappedPipelineScope) TableName() string { + return p.Scope.TableName +} diff --git a/backend/server/services/remote/models/plugin_remote.go b/backend/server/services/remote/models/plugin_remote.go new file mode 100644 index 00000000000..72f4a45bf99 --- /dev/null +++ b/backend/server/services/remote/models/plugin_remote.go @@ -0,0 +1,31 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// RemotePlugin API supported by plugins running in different/remote processes +type RemotePlugin interface { + plugin.PluginApi + plugin.PluginTask + plugin.PluginMeta + RunMigrations(forceMigrate bool) errors.Error +} diff --git a/backend/server/services/remote/plugin/connection_api.go b/backend/server/services/remote/plugin/connection_api.go new file mode 100644 index 00000000000..94b09fbee7c --- /dev/null +++ b/backend/server/services/remote/plugin/connection_api.go @@ -0,0 +1,92 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/server/services/remote/bridge" + "net/http" +) + +type ConnectionAPI struct { + invoker bridge.Invoker + connType *models.DynamicTabler + helper *api.ConnectionApiHelper +} + +func (c *ConnectionAPI) TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + err := c.invoker.Call("test-connection", bridge.DefaultContext, input.Body).Get() + if err != nil { + return nil, err + } + return nil, nil +} + +func (c *ConnectionAPI) PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := c.connType.New() + err := c.helper.Create(connection, input) + if err != nil { + return nil, err + } + conn := connection.Unwrap() + return &plugin.ApiResourceOutput{Body: conn, Status: http.StatusOK}, nil +} + +func (c *ConnectionAPI) ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connections := c.connType.NewSlice() + err := c.helper.List(connections) + if err != nil { + return nil, err + } + conns := connections.Unwrap() + return &plugin.ApiResourceOutput{Body: conns}, nil +} + +func (c *ConnectionAPI) GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := c.connType.New() + err := c.helper.First(connection, input.Params) + if err != nil { + return nil, err + } + conn := connection.Unwrap() + return &plugin.ApiResourceOutput{Body: conn}, nil +} + +func (c *ConnectionAPI) PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := c.connType.New() + err := c.helper.Patch(connection, input) + if err != nil { + return nil, err + } + conn := connection.Unwrap() + return &plugin.ApiResourceOutput{Body: conn, Status: http.StatusOK}, nil +} + +func (c *ConnectionAPI) DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := c.connType.New() + err := c.helper.First(connection, input.Params) + if err != nil { + return nil, err + } + err = c.helper.Delete(connection) + conn := connection.Unwrap() + return &plugin.ApiResourceOutput{Body: conn}, err +} diff --git a/backend/server/services/remote/plugin/default_api.go b/backend/server/services/remote/plugin/default_api.go new file mode 100644 index 00000000000..a2d500b2884 --- /dev/null +++ b/backend/server/services/remote/plugin/default_api.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/server/services/remote/bridge" +) + +func GetDefaultAPI(invoker bridge.Invoker, connType *models.DynamicTabler, helper *api.ConnectionApiHelper) map[string]map[string]plugin.ApiResourceHandler { + connectionApi := &ConnectionAPI{ + invoker: invoker, + connType: connType, + helper: helper, + } + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": connectionApi.TestConnection, + }, + "connections": { + "POST": connectionApi.PostConnections, + "GET": connectionApi.ListConnections, + }, + "connections/:connectionId": { + "GET": connectionApi.GetConnection, + "PATCH": connectionApi.PatchConnection, + "DELETE": connectionApi.DeleteConnection, + }, + } +} diff --git a/backend/server/services/remote/plugin/init.go b/backend/server/services/remote/plugin/init.go new file mode 100644 index 00000000000..1ddd03001c0 --- /dev/null +++ b/backend/server/services/remote/plugin/init.go @@ -0,0 +1,61 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/server/services/remote/bridge" + "github.com/apache/incubator-devlake/server/services/remote/models" + "github.com/go-playground/validator/v10" +) + +var ( + connectionHelper *api.ConnectionApiHelper + basicRes context.BasicRes +) + +func Init(br context.BasicRes) { + vld := validator.New() + basicRes = br + connectionHelper = api.NewConnectionHelper( + br, + vld, + ) +} + +func NewRemotePlugin(info *models.PluginInfo) (models.RemotePlugin, errors.Error) { + invoker := bridge.NewPythonPoetryCmdInvoker(info.PluginPath) + plugin, err := newPlugin(info, invoker) + + if err != nil { + return nil, err + } + + switch info.Extension { + case models.None: + return plugin, nil + case models.Metric: + return remoteMetricPlugin{plugin}, nil + case models.Datasource: + return remoteDatasourcePlugin{plugin}, nil + default: + return nil, errors.BadInput.New("unsupported plugin extension") + } +} diff --git a/backend/server/services/remote/plugin/plugin_extensions.go b/backend/server/services/remote/plugin/plugin_extensions.go new file mode 100644 index 00000000000..537e4640fbb --- /dev/null +++ b/backend/server/services/remote/plugin/plugin_extensions.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/server/services/remote/bridge" + "github.com/apache/incubator-devlake/server/services/remote/models" +) + +type ( + remoteMetricPlugin struct { + *remotePluginImpl + } + remoteDatasourcePlugin struct { + *remotePluginImpl + } +) + +func (p *remoteMetricPlugin) MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (plugin.PipelinePlan, errors.Error) { + plan := plugin.PipelinePlan{} + err := p.invoker.Call("MakeMetricPluginPipelinePlanV200", bridge.DefaultContext, projectName, options).Get(&plan) + if err != nil { + return nil, err + } + return plan, err +} + +func (p *remoteDatasourcePlugin) MakeDataSourcePipelinePlanV200(connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, errors.Error) { + plan := plugin.PipelinePlan{} + var scopes []models.PipelineScope + err := p.invoker.Call("make-pipeline", bridge.DefaultContext, connectionId, bpScopes).Get(&plan, &scopes) + if err != nil { + return nil, nil, err + } + var castedScopes []plugin.Scope + for _, scope := range scopes { + castedScopes = append(castedScopes, &models.WrappedPipelineScope{Scope: scope}) + } + return plan, castedScopes, nil +} + +var _ models.RemotePlugin = (*remoteMetricPlugin)(nil) +var _ plugin.MetricPluginBlueprintV200 = (*remoteMetricPlugin)(nil) +var _ models.RemotePlugin = (*remoteDatasourcePlugin)(nil) +var _ plugin.DataSourcePluginBlueprintV200 = (*remoteDatasourcePlugin)(nil) diff --git a/backend/server/services/remote/plugin/plugin_impl.go b/backend/server/services/remote/plugin/plugin_impl.go new file mode 100644 index 00000000000..7e611b2837e --- /dev/null +++ b/backend/server/services/remote/plugin/plugin_impl.go @@ -0,0 +1,131 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/server/services/remote/bridge" + "github.com/apache/incubator-devlake/server/services/remote/models" +) + +type ( + remotePluginImpl struct { + name string + subtaskMetas []plugin.SubTaskMeta + pluginPath string + description string + invoker bridge.Invoker + connectionTabler *coreModels.DynamicTabler + resources map[string]map[string]plugin.ApiResourceHandler + } + RemotePluginTaskData struct { + DbUrl string `json:"db_url"` + ConnectionId uint64 `json:"connection_id"` + Connection interface{} `json:"connection"` + Options map[string]interface{} `json:"options"` + } +) + +func newPlugin(info *models.PluginInfo, invoker bridge.Invoker) (*remotePluginImpl, errors.Error) { + connectionTableName := fmt.Sprintf("_tool_%s_connections", info.Name) + dynamicTabler, err := models.LoadTableModel(connectionTableName, info.ConnectionSchema) + if err != nil { + return nil, err + } + p := remotePluginImpl{ + name: info.Name, + invoker: invoker, + pluginPath: info.PluginPath, + description: info.Description, + connectionTabler: dynamicTabler, + resources: GetDefaultAPI(invoker, dynamicTabler, connectionHelper), + } + remoteBridge := bridge.NewBridge(invoker) + for _, subtask := range info.SubtaskMetas { + p.subtaskMetas = append(p.subtaskMetas, plugin.SubTaskMeta{ + Name: subtask.Name, + EntryPoint: remoteBridge.RemoteSubtaskEntrypointHandler(subtask), + Required: subtask.Required, + EnabledByDefault: subtask.EnabledByDefault, + Description: subtask.Description, + DomainTypes: subtask.DomainTypes, + }) + } + return &p, nil +} + +func (p *remotePluginImpl) SubTaskMetas() []plugin.SubTaskMeta { + return p.subtaskMetas +} + +func (p *remotePluginImpl) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + dbUrl := taskCtx.GetConfig("db_url") + connectionId := uint64(options["connectionId"].(float64)) + + connectionHelper := api.NewConnectionHelper( + taskCtx, + nil, + ) + + connection := p.connectionTabler.New() + err := connectionHelper.FirstById(connection, connectionId) + if err != nil { + return nil, errors.Convert(err) + } + + return RemotePluginTaskData{ + DbUrl: dbUrl, + ConnectionId: connectionId, + Connection: connection.Unwrap(), + Options: options, + }, nil +} + +func (p *remotePluginImpl) Description() string { + return p.description +} + +func (p *remotePluginImpl) RootPkgPath() string { + // RootPkgPath is only used to find to which plugin a given type belongs. + // This in turn is only used by DomainIdGenerator. + // Remote plugins define tool layer types in another language, + // so the reflective implementation of NewDomainIdGenerator cannot work. + return "" +} + +func (p *remotePluginImpl) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return p.resources +} + +func (p *remotePluginImpl) RunMigrations(forceMigrate bool) errors.Error { + err := api.CallDB(basicRes.GetDal().AutoMigrate, p.connectionTabler.New()) + if err != nil { + return err + } + + err = p.invoker.Call("run-migrations", bridge.DefaultContext, forceMigrate).Get() + return err +} + +var _ models.RemotePlugin = (*remotePluginImpl)(nil) +var _ plugin.Scope = (*models.WrappedPipelineScope)(nil) diff --git a/backend/server/services/remote/run/run.go b/backend/server/services/remote/run/run.go new file mode 100644 index 00000000000..b774ab1f875 --- /dev/null +++ b/backend/server/services/remote/run/run.go @@ -0,0 +1,65 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "encoding/json" + "fmt" + + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/server/services/remote" + "github.com/apache/incubator-devlake/server/services/remote/bridge" + "github.com/apache/incubator-devlake/server/services/remote/models" + "github.com/spf13/cobra" +) + +func main() { + cmd := &cobra.Command{Use: "run"} + + pluginPath := cmd.Flags().StringP("path", "p", "", "path of the plugin directory") + connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "connection id") + optionsJSON := cmd.Flags().StringP("options", "o", "{}", "plugin options as a JSON object") + _ = cmd.MarkFlagRequired("path") + _ = cmd.MarkFlagRequired("connectionId") + + cmd.Run = func(cmd *cobra.Command, args []string) { + invoker := bridge.NewPythonPoetryCmdInvoker(*pluginPath) + + pluginInfo := models.PluginInfo{} + err := invoker.Call("plugin-info", bridge.DefaultContext).Get(&pluginInfo) + + if err != nil { + panic(fmt.Sprintf("Cannot get plugin info: %s", err)) + } + + plugin, err := remote.NewRemotePlugin(&pluginInfo) + if err != nil { + panic(fmt.Sprintf("Cannot initialize plugin: %s", err)) + } + + var options map[string]interface{} + jsonErr := json.Unmarshal([]byte(*optionsJSON), &options) + if jsonErr != nil { + panic(fmt.Sprintf("Cannot parse options: %s", jsonErr)) + } + + options["connectionId"] = *connectionId + runner.DirectRun(cmd, args, plugin, options) + } + runner.RunCmd(cmd) +} diff --git a/backend/test/helper/api.go b/backend/test/helper/api.go new file mode 100644 index 00000000000..a29b987a720 --- /dev/null +++ b/backend/test/helper/api.go @@ -0,0 +1,253 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import ( + "fmt" + "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + apiProject "github.com/apache/incubator-devlake/server/api/project" + "net/http" + "reflect" + "time" +) + +type Connection struct { + api.BaseConnection `mapstructure:",squash"` + api.RestConnection `mapstructure:",squash"` + api.AccessToken `mapstructure:",squash"` +} + +// CreateConnection FIXME +func (d *DevlakeClient) TestConnection(pluginName string, connection any) { + d.testCtx.Helper() + _ = sendHttpRequest[Connection](d.testCtx, d.timeout, debugInfo{ + print: true, + inlineJson: false, + }, http.MethodPost, fmt.Sprintf("%s/plugins/%s/test", d.Endpoint, pluginName), connection) +} + +// CreateConnection FIXME +func (d *DevlakeClient) CreateConnection(pluginName string, connection any) *Connection { + d.testCtx.Helper() + created := sendHttpRequest[Connection](d.testCtx, d.timeout, debugInfo{ + print: true, + inlineJson: false, + }, http.MethodPost, fmt.Sprintf("%s/plugins/%s/connections", d.Endpoint, pluginName), connection) + return &created +} + +// ListConnections FIXME +func (d *DevlakeClient) ListConnections(pluginName string) []*Connection { + d.testCtx.Helper() + all := sendHttpRequest[[]*Connection](d.testCtx, d.timeout, debugInfo{ + print: true, + inlineJson: false, + }, http.MethodGet, fmt.Sprintf("%s/plugins/%s/connections", d.Endpoint, pluginName), nil) + return all +} + +type BlueprintV2Config struct { + Connection *plugin.BlueprintConnectionV200 + CreatedDateAfter *time.Time + SkipOnFail bool + ProjectName string +} + +// CreateBasicBlueprintV2 FIXME +func (d *DevlakeClient) CreateBasicBlueprintV2(name string, config *BlueprintV2Config) models.Blueprint { + settings := &models.BlueprintSettings{ + Version: "2.0.0", + CreatedDateAfter: config.CreatedDateAfter, + Connections: ToJson([]*plugin.BlueprintConnectionV200{ + config.Connection, + }), + } + blueprint := models.Blueprint{ + Name: name, + ProjectName: config.ProjectName, + Mode: models.BLUEPRINT_MODE_NORMAL, + Plan: nil, + Enable: true, + CronConfig: "manual", + IsManual: true, + SkipOnFail: config.SkipOnFail, + Labels: []string{"test-label"}, + Settings: ToJson(settings), + } + d.testCtx.Helper() + blueprint = sendHttpRequest[models.Blueprint](d.testCtx, d.timeout, debugInfo{ + print: true, + inlineJson: false, + }, http.MethodPost, fmt.Sprintf("%s/blueprints", d.Endpoint), &blueprint) + return blueprint +} + +type ( + ProjectPlugin struct { + Name string + Options any + } + ProjectConfig struct { + ProjectName string + ProjectDescription string + EnableDora bool + MetricPlugins []ProjectPlugin + } +) + +func (d *DevlakeClient) CreateProject(project *ProjectConfig) models.ApiOutputProject { + var metrics []models.BaseMetric + doraSeen := false + for _, p := range project.MetricPlugins { + if p.Name == "dora" { + doraSeen = true + } + metrics = append(metrics, models.BaseMetric{ + PluginName: p.Name, + PluginOption: string(ToJson(p.Options)), + Enable: true, + }) + } + if project.EnableDora && !doraSeen { + metrics = append(metrics, models.BaseMetric{ + PluginName: "dora", + PluginOption: string(ToJson(nil)), + Enable: true, + }) + } + return sendHttpRequest[models.ApiOutputProject](d.testCtx, d.timeout, debugInfo{ + print: true, + inlineJson: false, + }, http.MethodPost, fmt.Sprintf("%s/projects", d.Endpoint), &models.ApiInputProject{ + BaseProject: models.BaseProject{ + Name: project.ProjectName, + Description: project.ProjectDescription, + }, + Enable: Val(true), + Metrics: &metrics, + }) +} + +func (d *DevlakeClient) GetProject(projectName string) models.ApiOutputProject { + return sendHttpRequest[models.ApiOutputProject](d.testCtx, d.timeout, debugInfo{ + print: true, + inlineJson: false, + }, http.MethodGet, fmt.Sprintf("%s/projects/%s", d.Endpoint, projectName), nil) +} + +func (d *DevlakeClient) ListProjects() apiProject.PaginatedProjects { + return sendHttpRequest[apiProject.PaginatedProjects](d.testCtx, d.timeout, debugInfo{ + print: true, + inlineJson: false, + }, http.MethodGet, fmt.Sprintf("%s/projects", d.Endpoint), nil) +} + +func (d *DevlakeClient) CreateScope(pluginName string, connectionId uint64, scope any) any { + return sendHttpRequest[any](d.testCtx, d.timeout, debugInfo{ + print: true, + inlineJson: false, + }, http.MethodPut, fmt.Sprintf("%s/plugins/%s/connections/%d/scopes", d.Endpoint, pluginName, connectionId), scope) +} + +func (d *DevlakeClient) ListScopes(pluginName string, connectionId uint64) []any { + return sendHttpRequest[[]any](d.testCtx, d.timeout, debugInfo{ + print: true, + inlineJson: false, + }, http.MethodGet, fmt.Sprintf("%s/plugins/%s/connections/%d/scopes", d.Endpoint, pluginName, connectionId), nil) +} + +func (d *DevlakeClient) CreateTransformRule(pluginName string, rules any) any { + return sendHttpRequest[any](d.testCtx, d.timeout, debugInfo{ + print: true, + inlineJson: false, + }, http.MethodPost, fmt.Sprintf("%s/plugins/%s/transformation_rules", d.Endpoint, pluginName), rules) +} + +func (d *DevlakeClient) ListTransformRules(pluginName string) []any { + return sendHttpRequest[[]any](d.testCtx, d.timeout, debugInfo{ + print: true, + inlineJson: false, + }, http.MethodGet, fmt.Sprintf("%s/plugins/%s/transformation_rules?pageSize=20?page=1", d.Endpoint, pluginName), nil) +} + +// CreateBasicBlueprint FIXME +func (d *DevlakeClient) CreateBasicBlueprint(name string, connection *plugin.BlueprintConnectionV100) models.Blueprint { + settings := &models.BlueprintSettings{ + Version: "1.0.0", + Connections: ToJson([]*plugin.BlueprintConnectionV100{connection}), + } + blueprint := models.Blueprint{ + Name: name, + Mode: models.BLUEPRINT_MODE_NORMAL, + Plan: nil, + Enable: true, + CronConfig: "manual", + IsManual: true, + Settings: ToJson(settings), + } + d.testCtx.Helper() + blueprint = sendHttpRequest[models.Blueprint](d.testCtx, d.timeout, debugInfo{ + print: true, + inlineJson: false, + }, http.MethodPost, fmt.Sprintf("%s/blueprints", d.Endpoint), &blueprint) + return blueprint +} + +// TriggerBlueprint FIXME +func (d *DevlakeClient) TriggerBlueprint(blueprintId uint64) models.Pipeline { + d.testCtx.Helper() + pipeline := sendHttpRequest[models.Pipeline](d.testCtx, d.timeout, debugInfo{ + print: true, + inlineJson: false, + }, http.MethodPost, fmt.Sprintf("%s/blueprints/%d/trigger", d.Endpoint, blueprintId), nil) + return d.monitorPipeline(pipeline.ID) +} + +// RunPipeline FIXME +func (d *DevlakeClient) RunPipeline(pipeline models.NewPipeline) models.Pipeline { + d.testCtx.Helper() + pipelineResult := sendHttpRequest[models.Pipeline](d.testCtx, d.timeout, debugInfo{ + print: true, + inlineJson: false, + }, http.MethodPost, fmt.Sprintf("%s/pipelines", d.Endpoint), &pipeline) + return d.monitorPipeline(pipelineResult.ID) +} + +// MonitorPipeline FIXME +func (d *DevlakeClient) monitorPipeline(id uint64) models.Pipeline { + d.testCtx.Helper() + var previousResult models.Pipeline + endpoint := fmt.Sprintf("%s/pipelines/%d", d.Endpoint, id) + coloredPrintf("calling:\n\t%s %s\nwith:\n%s\n", http.MethodGet, endpoint, string(ToCleanJson(false, nil))) + for { + time.Sleep(1 * time.Second) + pipelineResult := sendHttpRequest[models.Pipeline](d.testCtx, d.timeout, debugInfo{ + print: false, + }, http.MethodGet, fmt.Sprintf("%s/pipelines/%d", d.Endpoint, id), nil) + if pipelineResult.Status == models.TASK_COMPLETED || pipelineResult.Status == models.TASK_FAILED { + coloredPrintf("result: %s\n", ToCleanJson(true, &pipelineResult)) + return pipelineResult + } + if !reflect.DeepEqual(pipelineResult, previousResult) { + coloredPrintf("result: %s\n", ToCleanJson(true, &pipelineResult)) + } + previousResult = pipelineResult + } +} diff --git a/backend/test/helper/client.go b/backend/test/helper/client.go new file mode 100644 index 00000000000..fa3bb2428be --- /dev/null +++ b/backend/test/helper/client.go @@ -0,0 +1,307 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import ( + "bytes" + "context" + "encoding/json" + goerror "errors" + "fmt" + "io" + "net/http" + "os" + "syscall" + "testing" + "time" + + "github.com/apache/incubator-devlake/core/config" + corectx "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/migration" + "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/runner" + contextimpl "github.com/apache/incubator-devlake/impls/context" + "github.com/apache/incubator-devlake/impls/dalgorm" + "github.com/apache/incubator-devlake/impls/logruslog" + "github.com/apache/incubator-devlake/server/api" + remotePlugin "github.com/apache/incubator-devlake/server/services/remote/plugin" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +var throwawayDir string + +func init() { + tempDir, err := errors.Convert01(os.MkdirTemp("", "devlake_test"+"_*")) + if err != nil { + panic(err) + } + throwawayDir = tempDir +} + +// DevlakeClient FIXME +type ( + DevlakeClient struct { + Endpoint string + db *gorm.DB + log log.Logger + cfg *viper.Viper + testCtx *testing.T + basicRes corectx.BasicRes + timeout time.Duration + } + LocalClientConfig struct { + ServerPort uint + DbURL string + CreateServer bool + DropDb bool + Plugins map[string]plugin.PluginMeta + AdditionalMigrations func() []plugin.MigrationScript + Timeout time.Duration + } + RemoteClientConfig struct { + Endpoint string + } +) + +// ConnectRemoteServer returns a client to an existing server based on the config +func ConnectRemoteServer(t *testing.T, sbConfig *RemoteClientConfig) *DevlakeClient { + return &DevlakeClient{ + Endpoint: sbConfig.Endpoint, + db: nil, + log: nil, + testCtx: t, + } +} + +// ConnectLocalServer spins up a local server from the config and returns a client connected to it +func ConnectLocalServer(t *testing.T, sbConfig *LocalClientConfig) *DevlakeClient { + t.Helper() + fmt.Printf("Using test temp directory: %s\n", throwawayDir) + logger := logruslog.Global.Nested("test") + cfg := config.GetConfig() + cfg.Set("DB_URL", sbConfig.DbURL) + db, err := runner.NewGormDb(cfg, logger) + require.NoError(t, err) + addr := fmt.Sprintf("http://localhost:%d", sbConfig.ServerPort) + d := &DevlakeClient{ + Endpoint: addr, + db: db, + log: logger, + cfg: cfg, + basicRes: contextimpl.NewDefaultBasicRes(cfg, logger, dalgorm.NewDalgorm(db)), + testCtx: t, + timeout: sbConfig.Timeout, + } + d.configureEncryption() + d.initPlugins(sbConfig) + if sbConfig.DropDb { + d.dropDB() + } + if sbConfig.CreateServer { + cfg.Set("PORT", fmt.Sprintf(":%d", sbConfig.ServerPort)) + cfg.Set("PLUGIN_DIR", throwawayDir) + cfg.Set("LOGGING_DIR", throwawayDir) + go func() { + api.CreateApiService() + }() + } + req, err2 := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/proceed-db-migration", addr), nil) + require.NoError(t, err2) + d.forceSendHttpRequest(20, req, func(err errors.Error) bool { + e := err.Unwrap() + return goerror.Is(e, syscall.ECONNREFUSED) + }) + d.runMigrations(sbConfig) + return d +} + +// SetTimeout override the timeout of api requests +func (d *DevlakeClient) SetTimeout(timeout time.Duration) { + d.timeout = timeout +} + +// RunPlugin manually execute a plugin directly (local server only) +func (d *DevlakeClient) RunPlugin(ctx context.Context, pluginName string, pluginTask plugin.PluginTask, options map[string]interface{}, subtaskNames ...string) errors.Error { + if len(subtaskNames) == 0 { + subtaskNames = GetSubtaskNames(pluginTask.SubTaskMetas()...) + } + optionsJson, err := json.Marshal(options) + if err != nil { + return errors.Convert(err) + } + subtasksJson, err := json.Marshal(subtaskNames) + if err != nil { + return errors.Convert(err) + } + task := &models.Task{ + Plugin: pluginName, + Options: string(optionsJson), + Subtasks: subtasksJson, + } + return runner.RunPluginSubTasks( + ctx, + d.basicRes, + task, + pluginTask, + nil, + ) +} + +func (d *DevlakeClient) configureEncryption() { + v := config.GetConfig() + encKey := v.GetString(plugin.EncodeKeyEnvStr) + if encKey == "" { + // Randomly generate a bunch of encryption keys and set them to config + encKey = plugin.RandomEncKey() + v.Set(plugin.EncodeKeyEnvStr, encKey) + err := config.WriteConfig(v) + if err != nil { + panic(err) + } + } +} + +func (d *DevlakeClient) forceSendHttpRequest(retries uint, req *http.Request, onError func(err errors.Error) bool) { + d.testCtx.Helper() + for { + res, err := http.DefaultClient.Do(req) + if err != nil { + if !onError(errors.Default.WrapRaw(err)) { + require.NoError(d.testCtx, err) + } + } else { + if res.StatusCode != http.StatusOK { + panic(fmt.Sprintf("received HTTP status %d", res.StatusCode)) + } + return + } + retries-- + if retries == 0 { + panic("retry limit exceeded") + } + fmt.Printf("retrying http call to %s\n", req.URL.String()) + time.Sleep(1 * time.Second) + } +} + +func (d *DevlakeClient) initPlugins(sbConfig *LocalClientConfig) { + remotePlugin.Init(d.basicRes) + d.testCtx.Helper() + if sbConfig.Plugins != nil { + for name, p := range sbConfig.Plugins { + require.NoError(d.testCtx, plugin.RegisterPlugin(name, p)) + } + } + for _, p := range plugin.AllPlugins() { + if pi, ok := p.(plugin.PluginInit); ok { + err := pi.Init(d.basicRes) + require.NoError(d.testCtx, err) + } + } +} + +func (d *DevlakeClient) runMigrations(sbConfig *LocalClientConfig) { + d.testCtx.Helper() + basicRes := contextimpl.NewDefaultBasicRes(d.cfg, d.log, dalgorm.NewDalgorm(d.db)) + getMigrator := func() plugin.Migrator { + migrator, err := migration.NewMigrator(basicRes) + require.NoError(d.testCtx, err) + return migrator + } + { + migrator := getMigrator() + for pluginName, pluginInst := range sbConfig.Plugins { + if migratable, ok := pluginInst.(plugin.PluginMigration); ok { + migrator.Register(migratable.MigrationScripts(), pluginName) + } + } + require.NoError(d.testCtx, migrator.Execute()) + } + { + migrator := getMigrator() + if sbConfig.AdditionalMigrations != nil { + scripts := sbConfig.AdditionalMigrations() + migrator.Register(scripts, "extra migrations") + } + require.NoError(d.testCtx, migrator.Execute()) + } +} + +func (d *DevlakeClient) dropDB() { + d.testCtx.Helper() + migrator := d.db.Migrator() + tables, err := migrator.GetTables() + require.NoError(d.testCtx, err) + var tablesRaw []any + for _, table := range tables { + tablesRaw = append(tablesRaw, table) + } + err = migrator.DropTable(tablesRaw...) + require.NoError(d.testCtx, err) +} + +func sendHttpRequest[Res any](t *testing.T, timeout time.Duration, debug debugInfo, httpMethod string, endpoint string, body any) Res { + t.Helper() + b := ToJson(body) + if debug.print { + coloredPrintf("calling:\n\t%s %s\nwith:\n%s\n", httpMethod, endpoint, string(ToCleanJson(debug.inlineJson, body))) + } + timer := time.After(timeout) + for { + request, err := http.NewRequest(httpMethod, endpoint, bytes.NewReader(b)) + require.NoError(t, err) + request.Header.Add("Content-Type", "application/json") + response, err := http.DefaultClient.Do(request) + require.NoError(t, err) + if timeout > 0 { + select { + case <-timer: + default: + if response.StatusCode >= 300 { + time.Sleep(1 * time.Second) + continue + } + } + } + require.True(t, response.StatusCode < 300, "unexpected http status code: %d", response.StatusCode) + var result Res + b, _ = io.ReadAll(response.Body) + require.NoError(t, json.Unmarshal(b, &result)) + if debug.print { + coloredPrintf("result: %s\n", ToCleanJson(debug.inlineJson, b)) + } + require.NoError(t, response.Body.Close()) + return result + } +} + +func coloredPrintf(msg string, args ...any) { + msg = fmt.Sprintf(msg, args...) + colorifier := "\033[1;33m%+v\033[0m" //yellow + fmt.Printf(colorifier, msg) +} + +type debugInfo struct { + print bool + inlineJson bool +} diff --git a/backend/test/helper/db_selector.go b/backend/test/helper/db_selector.go new file mode 100644 index 00000000000..9cbe2356940 --- /dev/null +++ b/backend/test/helper/db_selector.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import ( + "fmt" + "os" + + "github.com/apache/incubator-devlake/core/config" +) + +// UseMySQL FIXME +func UseMySQL(host string, port int) string { + conn := fmt.Sprintf("mysql://merico:merico@%s:%d/lake?charset=utf8mb4&parseTime=True", host, port) + _ = os.Setenv("E2E_DB_URL", conn) + config.GetConfig().Set("E2E_DB_URL", conn) + return conn +} + +// UsePostgres FIXME +func UsePostgres(host string, port int) string { + conn := fmt.Sprintf("postgres://merico:merico@%s:%d/lake", host, port) + _ = os.Setenv("E2E_DB_URL", conn) + config.GetConfig().Set("E2E_DB_URL", conn) + return conn +} diff --git a/backend/test/helper/init.go b/backend/test/helper/init.go new file mode 100644 index 00000000000..0ba99d91fdd --- /dev/null +++ b/backend/test/helper/init.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import ( + "os" + "path/filepath" +) + +var ( + ProjectRoot = "" + Shell = "" +) + +func LocalInit() { + pwd, err := os.Getwd() + if err != nil { + panic(err) + } + for { + dir := filepath.Base(pwd) + if dir == "" { + panic("base repo directory not found") + } + if dir == "lake" || dir == "incubator-devlake" { + ProjectRoot = pwd + break + } + pwd = filepath.Dir(pwd) + } + Shell = "/bin/sh" + err = os.Chdir(ProjectRoot) + if err != nil { + panic(err) + } +} diff --git a/backend/test/helper/json_helper.go b/backend/test/helper/json_helper.go new file mode 100644 index 00000000000..2ce2cdd0cb4 --- /dev/null +++ b/backend/test/helper/json_helper.go @@ -0,0 +1,104 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import ( + "encoding/json" + "reflect" + "time" +) + +// ToJson FIXME +func ToJson(x any) json.RawMessage { + b, err := json.Marshal(x) + if err != nil { + panic(err) + } + return b +} + +// ToCleanJson FIXME +func ToCleanJson(inline bool, x any) json.RawMessage { + j, err := json.Marshal(x) + if err != nil { + panic(err) + } + var m any + if j[0] == '[' { + //it's a slice + m = x + } else { + m = map[string]any{} + err = json.Unmarshal(j, &m) + if err != nil { + panic(err) + } + if m != nil { + removeNullsFromMap(m.(map[string]any)) + } + } + var b []byte + if inline { + b, err = json.Marshal(m) + } else { + b, err = json.MarshalIndent(m, "", " ") + } + if err != nil { + panic(err) + } + return b +} + +func removeNullsFromMap(m map[string]any) { + refMap := reflect.ValueOf(m) + for _, refKey := range refMap.MapKeys() { + key := refKey.String() + refValue := refMap.MapIndex(refKey) + if refValue.IsNil() || refValue.Elem().IsZero() { + delete(m, key) + continue + } + value := refValue.Interface() + if isNullTime(value) { + delete(m, key) + continue + } + switch valueCasted := value.(type) { + case map[string]any: + removeNullsFromMap(valueCasted) + case []any: + for _, arrayValue := range valueCasted { + if m, ok := arrayValue.(map[string]any); ok { + removeNullsFromMap(m) + } + } + } + } +} + +func isNullTime(value any) bool { + if str, ok := value.(string); ok { + if t, err := time.Parse("2006-01-02T15:04:05Z", str); err == nil { + zeroTime := time.Time{} + if t.Equal(zeroTime) { + return true + } + } + } + return false +} diff --git a/backend/test/helper/utils.go b/backend/test/helper/utils.go new file mode 100644 index 00000000000..479fcd53483 --- /dev/null +++ b/backend/test/helper/utils.go @@ -0,0 +1,91 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/apache/incubator-devlake/core/plugin" +) + +// IsWSL FIXME +func IsWSL() bool { + lines, err := readFile("/proc/version") + if err != nil { + return false + } + for _, line := range lines { + l := strings.ToLower(line) + if strings.Contains(l, "microsoft") { + return true + } + } + return false +} + +// GetSubtaskNames FIXME +func GetSubtaskNames(metas ...plugin.SubTaskMeta) []string { + var names []string + for _, m := range metas { + names = append(names, m.Name) + } + return names +} + +// AddToPath FIXME +func AddToPath(newPaths ...string) { + path := os.ExpandEnv("$PATH") + for _, newPath := range newPaths { + newPath, _ = filepath.Abs(newPath) + path = fmt.Sprintf("%s:%s", newPath, path) + } + _ = os.Setenv("PATH", path) +} + +func Val[T any](t T) *T { + return &t +} + +func Cast[T any](m any) T { + j := ToJson(m) + t := new(T) + err := json.Unmarshal(j, t) + if err != nil { + panic(err) + } + return *t +} + +func readFile(path string) ([]string, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + var lines []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + return lines, scanner.Err() +} diff --git a/backend/test/remote/docker-compose.test.yml b/backend/test/remote/docker-compose.test.yml new file mode 100644 index 00000000000..897b59ad5f0 --- /dev/null +++ b/backend/test/remote/docker-compose.test.yml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: "3" +services: + + mysql-test: + image: mysql:8.0.26 + platform: linux/x86_64 + volumes: + - mysql-test-storage:/var/lib/mysql + restart: always + ports: + - "3307:3306" + environment: + MYSQL_ROOT_PASSWORD: admin + MYSQL_DATABASE: lake + MYSQL_USER: merico + MYSQL_PASSWORD: merico + + postgres-test: + image: postgres:14.2-alpine + restart: always + ports: + - "3308:5432" + volumes: + - postgres-test-storage:/var/lib/postgresql/data + environment: + POSTGRES_DB: lake + POSTGRES_USER: merico + POSTGRES_PASSWORD: merico + +volumes: + mysql-test-storage: + postgres-test-storage: \ No newline at end of file diff --git a/backend/test/remote/fakeplugin/build.sh b/backend/test/remote/fakeplugin/build.sh new file mode 100755 index 00000000000..1a4a9d605e9 --- /dev/null +++ b/backend/test/remote/fakeplugin/build.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd "${0%/*}" # make sure we're in the correct dir + +poetry config virtualenvs.create true +echo "Installing dependencies of fake python plugin" &&\ +poetry install &&\ +exit_code=$? +if [ $exit_code != 0 ]; then + exit $exit_code +fi diff --git a/backend/test/remote/fakeplugin/fakeplugin/__init__.py b/backend/test/remote/fakeplugin/fakeplugin/__init__.py new file mode 100644 index 00000000000..65d64ce95c0 --- /dev/null +++ b/backend/test/remote/fakeplugin/fakeplugin/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/backend/test/remote/fakeplugin/fakeplugin/main.py b/backend/test/remote/fakeplugin/fakeplugin/main.py new file mode 100644 index 00000000000..0c7c2db8ec1 --- /dev/null +++ b/backend/test/remote/fakeplugin/fakeplugin/main.py @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum +from datetime import datetime +from typing import Optional + +from sqlmodel import Field + +from pydevlake import Plugin, Connection, Stream, ToolModel +from pydevlake.domain_layer.devops import CICDScope, CICDPipeline + + +VALID_TOKEN = "this_is_a_valid_token" +VALID_PROJECT = "this_is_a_valid_project" + + +class FakePipeline(ToolModel, table=True): + class State(Enum): + PENDING = "pending" + RUNNING = "running" + FAILURE = "failure" + SUCCESS = "success" + + id: str = Field(primary_key=True) + project: str + started_at: Optional[datetime] + finished_at: Optional[datetime] + state: State + + +class FakeStream(Stream): + tool_model = FakePipeline + domain_model = CICDPipeline + + fake_pipelines = [ + FakePipeline(id=1, project=VALID_PROJECT, state=FakePipeline.State.SUCCESS, started_at=datetime(2023, 1, 10, 11, 0, 0), finished_at=datetime(2023, 1, 10, 11, 3, 0)), + FakePipeline(id=2, project=VALID_PROJECT, state=FakePipeline.State.FAILURE, started_at=datetime(2023, 1, 10, 12, 0, 0), finished_at=datetime(2023, 1, 10, 12, 1, 30)), + FakePipeline(id=1, project=VALID_PROJECT, state=FakePipeline.State.PENDING), + ] + + def collect(self, state, context): + project = context.options['project'] + if project == VALID_PROJECT: + for p in self.fake_pipelines: + yield dict(p) + + def convert(self, pipeline: FakePipeline): + yield CICDPipeline( + name=pipeline.id, + status=self.convert_status(pipeline.state), + finished_date=pipeline.finished_at, + result=self.convert_result(pipeline.state), + duration_sec=self.duration(pipeline), + environment=[], + type=CICDPipeline.Type.CI + ) + + def convert_status(self, state: FakePipeline.State): + match state: + case FakePipeline.State.FAILURE | FakePipeline.State.SUCCESS: + return CICDPipeline.Status.DONE + case _: + return CICDPipeline.Status.IN_PROGRESS + + def convert_result(self, state: FakePipeline.State): + match state: + case FakePipeline.State.SUCCESS: + return CICDPipeline.Result.SUCCESS + case FakePipeline.State.FAILURE: + return CICDPipeline.Status.FAILURE + case _: + return None + + def duration(self, pipeline: FakePipeline): + if pipeline.finished_at: + return (pipeline.finished_at - pipeline.started_at).seconds + return None + + +class FakeConnection(Connection): + token: str + + +class FakePlugin(Plugin): + @property + def connection_type(self): + return FakeConnection + + def get_scopes(self, scope_name: str, connection: FakeConnection): + assert connection + yield CICDScope( + id=1, + name=scope_name, + url=f"http://fake.org/api/project/{scope_name}" + ) + + def test_connection(self, connection: FakeConnection): + if connection.token != VALID_TOKEN: + raise Exception("Invalid token") + + @property + def streams(self): + return [ + FakeStream + ] + + +if __name__ == '__main__': + FakePlugin.start() diff --git a/backend/test/remote/fakeplugin/poetry.lock b/backend/test/remote/fakeplugin/poetry.lock new file mode 100644 index 00000000000..ab0cac7b0e0 --- /dev/null +++ b/backend/test/remote/fakeplugin/poetry.lock @@ -0,0 +1,513 @@ +# This file is automatically @generated by Poetry and should not be changed by hand. + +[[package]] +name = "certifi" +version = "2022.12.7" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.0.1" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "charset-normalizer-3.0.1.tar.gz", hash = "sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db72b07027db150f468fbada4d85b3b2729a3db39178abf5c543b784c1254539"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62595ab75873d50d57323a91dd03e6966eb79c41fa834b7a1661ed043b2d404d"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:772b87914ff1152b92a197ef4ea40efe27a378606c39446ded52c8f80f79702e"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70990b9c51340e4044cfc394a81f614f3f90d41397104d226f21e66de668730d"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:292d5e8ba896bbfd6334b096e34bffb56161c81408d6d036a7dfa6929cff8783"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2edb64ee7bf1ed524a1da60cdcd2e1f6e2b4f66ef7c077680739f1641f62f555"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:31a9ddf4718d10ae04d9b18801bd776693487cbb57d74cc3458a7673f6f34639"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:44ba614de5361b3e5278e1241fda3dc1838deed864b50a10d7ce92983797fa76"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:12db3b2c533c23ab812c2b25934f60383361f8a376ae272665f8e48b88e8e1c6"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c512accbd6ff0270939b9ac214b84fb5ada5f0409c44298361b2f5e13f9aed9e"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-win32.whl", hash = "sha256:502218f52498a36d6bf5ea77081844017bf7982cdbe521ad85e64cabee1b608b"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:601f36512f9e28f029d9481bdaf8e89e5148ac5d89cffd3b05cd533eeb423b59"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0298eafff88c99982a4cf66ba2efa1128e4ddaca0b05eec4c456bbc7db691d8d"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8d0fc946c784ff7f7c3742310cc8a57c5c6dc31631269876a88b809dbeff3d3"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:87701167f2a5c930b403e9756fab1d31d4d4da52856143b609e30a1ce7160f3c"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14e76c0f23218b8f46c4d87018ca2e441535aed3632ca134b10239dfb6dadd6b"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c0a590235ccd933d9892c627dec5bc7511ce6ad6c1011fdf5b11363022746c1"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c7fe7afa480e3e82eed58e0ca89f751cd14d767638e2550c77a92a9e749c317"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79909e27e8e4fcc9db4addea88aa63f6423ebb171db091fb4373e3312cb6d603"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ac7b6a045b814cf0c47f3623d21ebd88b3e8cf216a14790b455ea7ff0135d18"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:72966d1b297c741541ca8cf1223ff262a6febe52481af742036a0b296e35fa5a"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5995f0164fa7df59db4746112fec3f49c461dd6b31b841873443bdb077c13cfc"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4a8fcf28c05c1f6d7e177a9a46a1c52798bfe2ad80681d275b10dcf317deaf0b"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:761e8904c07ad053d285670f36dd94e1b6ab7f16ce62b9805c475b7aa1cffde6"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-win32.whl", hash = "sha256:71140351489970dfe5e60fc621ada3e0f41104a5eddaca47a7acb3c1b851d6d3"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ab77acb98eba3fd2a85cd160851816bfce6871d944d885febf012713f06659c"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:84c3990934bae40ea69a82034912ffe5a62c60bbf6ec5bc9691419641d7d5c9a"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74292fc76c905c0ef095fe11e188a32ebd03bc38f3f3e9bcb85e4e6db177b7ea"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c95a03c79bbe30eec3ec2b7f076074f4281526724c8685a42872974ef4d36b72"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c39b0e3eac288fedc2b43055cfc2ca7a60362d0e5e87a637beac5d801ef478"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df2c707231459e8a4028eabcd3cfc827befd635b3ef72eada84ab13b52e1574d"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93ad6d87ac18e2a90b0fe89df7c65263b9a99a0eb98f0a3d2e079f12a0735837"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:59e5686dd847347e55dffcc191a96622f016bc0ad89105e24c14e0d6305acbc6"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:cd6056167405314a4dc3c173943f11249fa0f1b204f8b51ed4bde1a9cd1834dc"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:083c8d17153ecb403e5e1eb76a7ef4babfc2c48d58899c98fcaa04833e7a2f9a"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:f5057856d21e7586765171eac8b9fc3f7d44ef39425f85dbcccb13b3ebea806c"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:7eb33a30d75562222b64f569c642ff3dc6689e09adda43a082208397f016c39a"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:95dea361dd73757c6f1c0a1480ac499952c16ac83f7f5f4f84f0658a01b8ef41"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:eaa379fcd227ca235d04152ca6704c7cb55564116f8bc52545ff357628e10602"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3e45867f1f2ab0711d60c6c71746ac53537f1684baa699f4f668d4c6f6ce8e14"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cadaeaba78750d58d3cc6ac4d1fd867da6fc73c88156b7a3212a3cd4819d679d"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:911d8a40b2bef5b8bbae2e36a0b103f142ac53557ab421dc16ac4aafee6f53dc"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:503e65837c71b875ecdd733877d852adbc465bd82c768a067badd953bf1bc5a3"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a60332922359f920193b1d4826953c507a877b523b2395ad7bc716ddd386d866"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16a8663d6e281208d78806dbe14ee9903715361cf81f6d4309944e4d1e59ac5b"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a16418ecf1329f71df119e8a65f3aa68004a3f9383821edcb20f0702934d8087"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9d9153257a3f70d5f69edf2325357251ed20f772b12e593f3b3377b5f78e7ef8"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:02a51034802cbf38db3f89c66fb5d2ec57e6fe7ef2f4a44d070a593c3688667b"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:2e396d70bc4ef5325b72b593a72c8979999aa52fb8bcf03f701c1b03e1166918"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:11b53acf2411c3b09e6af37e4b9005cba376c872503c8f28218c7243582df45d"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:0bf2dae5291758b6f84cf923bfaa285632816007db0330002fa1de38bfcb7154"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2c03cc56021a4bd59be889c2b9257dae13bf55041a3372d3295416f86b295fb5"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:024e606be3ed92216e2b6952ed859d86b4cfa52cd5bc5f050e7dc28f9b43ec42"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4b0d02d7102dd0f997580b51edc4cebcf2ab6397a7edf89f1c73b586c614272c"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:358a7c4cb8ba9b46c453b1dd8d9e431452d5249072e4f56cfda3149f6ab1405e"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81d6741ab457d14fdedc215516665050f3822d3e56508921cc7239f8c8e66a58"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8b8af03d2e37866d023ad0ddea594edefc31e827fee64f8de5611a1dbc373174"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9cf4e8ad252f7c38dd1f676b46514f92dc0ebeb0db5552f5f403509705e24753"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e696f0dd336161fca9adbb846875d40752e6eba585843c768935ba5c9960722b"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c22d3fe05ce11d3671297dc8973267daa0f938b93ec716e12e0f6dee81591dc1"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:109487860ef6a328f3eec66f2bf78b0b72400280d8f8ea05f69c51644ba6521a"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:37f8febc8ec50c14f3ec9637505f28e58d4f66752207ea177c1d67df25da5aed"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f97e83fa6c25693c7a35de154681fcc257c1c41b38beb0304b9c4d2d9e164479"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a152f5f33d64a6be73f1d30c9cc82dfc73cec6477ec268e7c6e4c7d23c2d2291"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:39049da0ffb96c8cbb65cbf5c5f3ca3168990adf3551bd1dee10c48fce8ae820"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-win32.whl", hash = "sha256:4457ea6774b5611f4bed5eaa5df55f70abde42364d498c5134b7ef4c6958e20e"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:e62164b50f84e20601c1ff8eb55620d2ad25fb81b59e3cd776a1902527a788af"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8eade758719add78ec36dc13201483f8e9b5d940329285edcd5f70c0a9edbd7f"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8499ca8f4502af841f68135133d8258f7b32a53a1d594aa98cc52013fff55678"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3fc1c4a2ffd64890aebdb3f97e1278b0cc72579a08ca4de8cd2c04799a3a22be"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00d3ffdaafe92a5dc603cb9bd5111aaa36dfa187c8285c543be562e61b755f6b"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2ac1b08635a8cd4e0cbeaf6f5e922085908d48eb05d44c5ae9eabab148512ca"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6f45710b4459401609ebebdbcfb34515da4fc2aa886f95107f556ac69a9147e"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ae1de54a77dc0d6d5fcf623290af4266412a7c4be0b1ff7444394f03f5c54e3"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b590df687e3c5ee0deef9fc8c547d81986d9a1b56073d82de008744452d6541"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab5de034a886f616a5668aa5d098af2b5385ed70142090e2a31bcbd0af0fdb3d"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9cb3032517f1627cc012dbc80a8ec976ae76d93ea2b5feaa9d2a5b8882597579"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:608862a7bf6957f2333fc54ab4399e405baad0163dc9f8d99cb236816db169d4"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0f438ae3532723fb6ead77e7c604be7c8374094ef4ee2c5e03a3a17f1fca256c"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:356541bf4381fa35856dafa6a965916e54bed415ad8a24ee6de6e37deccf2786"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-win32.whl", hash = "sha256:39cf9ed17fe3b1bc81f33c9ceb6ce67683ee7526e65fde1447c772afc54a1bb8"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59"}, + {file = "charset_normalizer-3.0.1-py3-none-any.whl", hash = "sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24"}, +] + +[[package]] +name = "fire" +version = "0.4.0" +description = "A library for automatically generating command line interfaces." +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "fire-0.4.0.tar.gz", hash = "sha256:c5e2b8763699d1142393a46d0e3e790c5eb2f0706082df8f647878842c216a62"}, +] + +[package.dependencies] +six = "*" +termcolor = "*" + +[[package]] +name = "greenlet" +version = "2.0.2" +description = "Lightweight in-process concurrent programming" +category = "main" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" +files = [ + {file = "greenlet-2.0.2-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:bdfea8c661e80d3c1c99ad7c3ff74e6e87184895bbaca6ee8cc61209f8b9b85d"}, + {file = "greenlet-2.0.2-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:9d14b83fab60d5e8abe587d51c75b252bcc21683f24699ada8fb275d7712f5a9"}, + {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, + {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, + {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, + {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, + {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, + {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, + {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d75209eed723105f9596807495d58d10b3470fa6732dd6756595e89925ce2470"}, + {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3a51c9751078733d88e013587b108f1b7a1fb106d402fb390740f002b6f6551a"}, + {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, + {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, + {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, + {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, + {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, + {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, + {file = "greenlet-2.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:eff4eb9b7eb3e4d0cae3d28c283dc16d9bed6b193c2e1ace3ed86ce48ea8df19"}, + {file = "greenlet-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5454276c07d27a740c5892f4907c86327b632127dd9abec42ee62e12427ff7e3"}, + {file = "greenlet-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:7cafd1208fdbe93b67c7086876f061f660cfddc44f404279c1585bbf3cdc64c5"}, + {file = "greenlet-2.0.2-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:910841381caba4f744a44bf81bfd573c94e10b3045ee00de0cbf436fe50673a6"}, + {file = "greenlet-2.0.2-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:18a7f18b82b52ee85322d7a7874e676f34ab319b9f8cce5de06067384aa8ff43"}, + {file = "greenlet-2.0.2-cp35-cp35m-win32.whl", hash = "sha256:03a8f4f3430c3b3ff8d10a2a86028c660355ab637cee9333d63d66b56f09d52a"}, + {file = "greenlet-2.0.2-cp35-cp35m-win_amd64.whl", hash = "sha256:4b58adb399c4d61d912c4c331984d60eb66565175cdf4a34792cd9600f21b394"}, + {file = "greenlet-2.0.2-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:703f18f3fda276b9a916f0934d2fb6d989bf0b4fb5a64825260eb9bfd52d78f0"}, + {file = "greenlet-2.0.2-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:32e5b64b148966d9cccc2c8d35a671409e45f195864560829f395a54226408d3"}, + {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dd11f291565a81d71dab10b7033395b7a3a5456e637cf997a6f33ebdf06f8db"}, + {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e0f72c9ddb8cd28532185f54cc1453f2c16fb417a08b53a855c4e6a418edd099"}, + {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd021c754b162c0fb55ad5d6b9d960db667faad0fa2ff25bb6e1301b0b6e6a75"}, + {file = "greenlet-2.0.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:3c9b12575734155d0c09d6c3e10dbd81665d5c18e1a7c6597df72fd05990c8cf"}, + {file = "greenlet-2.0.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b9ec052b06a0524f0e35bd8790686a1da006bd911dd1ef7d50b77bfbad74e292"}, + {file = "greenlet-2.0.2-cp36-cp36m-win32.whl", hash = "sha256:dbfcfc0218093a19c252ca8eb9aee3d29cfdcb586df21049b9d777fd32c14fd9"}, + {file = "greenlet-2.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:9f35ec95538f50292f6d8f2c9c9f8a3c6540bbfec21c9e5b4b751e0a7c20864f"}, + {file = "greenlet-2.0.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:d5508f0b173e6aa47273bdc0a0b5ba055b59662ba7c7ee5119528f466585526b"}, + {file = "greenlet-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1"}, + {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9c59a2120b55788e800d82dfa99b9e156ff8f2227f07c5e3012a45a399620b7"}, + {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2780572ec463d44c1d3ae850239508dbeb9fed38e294c68d19a24d925d9223ca"}, + {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:937e9020b514ceedb9c830c55d5c9872abc90f4b5862f89c0887033ae33c6f73"}, + {file = "greenlet-2.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:36abbf031e1c0f79dd5d596bfaf8e921c41df2bdf54ee1eed921ce1f52999a86"}, + {file = "greenlet-2.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:18e98fb3de7dba1c0a852731c3070cf022d14f0d68b4c87a19cc1016f3bb8b33"}, + {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, + {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, + {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, + {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, + {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, + {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, + {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acd2162a36d3de67ee896c43effcd5ee3de247eb00354db411feb025aa319857"}, + {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0bf60faf0bc2468089bdc5edd10555bab6e85152191df713e2ab1fcc86382b5a"}, + {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, + {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, + {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, + {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, + {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, + {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, + {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be4ed120b52ae4d974aa40215fcdfde9194d63541c7ded40ee12eb4dda57b76b"}, + {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94c817e84245513926588caf1152e3b559ff794d505555211ca041f032abbb6b"}, + {file = "greenlet-2.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1a819eef4b0e0b96bb0d98d797bef17dc1b4a10e8d7446be32d1da33e095dbb8"}, + {file = "greenlet-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7efde645ca1cc441d6dc4b48c0f7101e8d86b54c8530141b09fd31cef5149ec9"}, + {file = "greenlet-2.0.2-cp39-cp39-win32.whl", hash = "sha256:ea9872c80c132f4663822dd2a08d404073a5a9b5ba6155bea72fb2a79d1093b5"}, + {file = "greenlet-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:db1a39669102a1d8d12b57de2bb7e2ec9066a6f2b3da35ae511ff93b01b5d564"}, + {file = "greenlet-2.0.2.tar.gz", hash = "sha256:e7c8dc13af7db097bed64a051d2dd49e9f0af495c26995c00a9ee842690d34c0"}, +] + +[package.extras] +docs = ["Sphinx", "docutils (<0.18)"] +test = ["objgraph", "psutil"] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] + +[[package]] +name = "inflect" +version = "6.0.2" +description = "Correctly generate plurals, singular nouns, ordinals, indefinite articles; convert numbers to words" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "inflect-6.0.2-py3-none-any.whl", hash = "sha256:182741ec7e9e4c8f7f55b01fa6d80bcd3c4a183d349dfa6d9abbff0a1279e98f"}, + {file = "inflect-6.0.2.tar.gz", hash = "sha256:f1a6bcb0105046f89619fde1a7d044c612c614c2d85ef182582d9dc9b86d309a"}, +] + +[package.dependencies] +pydantic = ">=1.9.1" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "pygments", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[[package]] +name = "mysqlclient" +version = "2.1.1" +description = "Python interface to MySQL" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ + {file = "mysqlclient-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:c1ed71bd6244993b526113cca3df66428609f90e4652f37eb51c33496d478b37"}, + {file = "mysqlclient-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:c812b67e90082a840efb82a8978369e6e69fc62ce1bda4ca8f3084a9d862308b"}, + {file = "mysqlclient-2.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:0d1cd3a5a4d28c222fa199002810e8146cffd821410b67851af4cc80aeccd97c"}, + {file = "mysqlclient-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:b355c8b5a7d58f2e909acdbb050858390ee1b0e13672ae759e5e784110022994"}, + {file = "mysqlclient-2.1.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:996924f3483fd36a34a5812210c69e71dea5a3d5978d01199b78b7f6d485c855"}, + {file = "mysqlclient-2.1.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:dea88c8d3f5a5d9293dfe7f087c16dd350ceb175f2f6631c9cf4caf3e19b7a96"}, + {file = "mysqlclient-2.1.1.tar.gz", hash = "sha256:828757e419fb11dd6c5ed2576ec92c3efaa93a0f7c39e263586d1ee779c3d782"}, +] + +[[package]] +name = "pydantic" +version = "1.10.4" +description = "Data validation and settings management using python type hints" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5635de53e6686fe7a44b5cf25fcc419a0d5e5c1a1efe73d49d48fe7586db854"}, + {file = "pydantic-1.10.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6dc1cc241440ed7ca9ab59d9929075445da6b7c94ced281b3dd4cfe6c8cff817"}, + {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51bdeb10d2db0f288e71d49c9cefa609bca271720ecd0c58009bd7504a0c464c"}, + {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78cec42b95dbb500a1f7120bdf95c401f6abb616bbe8785ef09887306792e66e"}, + {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8775d4ef5e7299a2f4699501077a0defdaac5b6c4321173bcb0f3c496fbadf85"}, + {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:572066051eeac73d23f95ba9a71349c42a3e05999d0ee1572b7860235b850cc6"}, + {file = "pydantic-1.10.4-cp310-cp310-win_amd64.whl", hash = "sha256:7feb6a2d401f4d6863050f58325b8d99c1e56f4512d98b11ac64ad1751dc647d"}, + {file = "pydantic-1.10.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39f4a73e5342b25c2959529f07f026ef58147249f9b7431e1ba8414a36761f53"}, + {file = "pydantic-1.10.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:983e720704431a6573d626b00662eb78a07148c9115129f9b4351091ec95ecc3"}, + {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d52162fe6b2b55964fbb0af2ee58e99791a3138588c482572bb6087953113a"}, + {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fdf8d759ef326962b4678d89e275ffc55b7ce59d917d9f72233762061fd04a2d"}, + {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05a81b006be15655b2a1bae5faa4280cf7c81d0e09fcb49b342ebf826abe5a72"}, + {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d88c4c0e5c5dfd05092a4b271282ef0588e5f4aaf345778056fc5259ba098857"}, + {file = "pydantic-1.10.4-cp311-cp311-win_amd64.whl", hash = "sha256:6a05a9db1ef5be0fe63e988f9617ca2551013f55000289c671f71ec16f4985e3"}, + {file = "pydantic-1.10.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:887ca463c3bc47103c123bc06919c86720e80e1214aab79e9b779cda0ff92a00"}, + {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdf88ab63c3ee282c76d652fc86518aacb737ff35796023fae56a65ced1a5978"}, + {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a48f1953c4a1d9bd0b5167ac50da9a79f6072c63c4cef4cf2a3736994903583e"}, + {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a9f2de23bec87ff306aef658384b02aa7c32389766af3c5dee9ce33e80222dfa"}, + {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:cd8702c5142afda03dc2b1ee6bc358b62b3735b2cce53fc77b31ca9f728e4bc8"}, + {file = "pydantic-1.10.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6e7124d6855b2780611d9f5e1e145e86667eaa3bd9459192c8dc1a097f5e9903"}, + {file = "pydantic-1.10.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b53e1d41e97063d51a02821b80538053ee4608b9a181c1005441f1673c55423"}, + {file = "pydantic-1.10.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:55b1625899acd33229c4352ce0ae54038529b412bd51c4915349b49ca575258f"}, + {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301d626a59edbe5dfb48fcae245896379a450d04baeed50ef40d8199f2733b06"}, + {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6f9d649892a6f54a39ed56b8dfd5e08b5f3be5f893da430bed76975f3735d15"}, + {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d7b5a3821225f5c43496c324b0d6875fde910a1c2933d726a743ce328fbb2a8c"}, + {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f2f7eb6273dd12472d7f218e1fef6f7c7c2f00ac2e1ecde4db8824c457300416"}, + {file = "pydantic-1.10.4-cp38-cp38-win_amd64.whl", hash = "sha256:4b05697738e7d2040696b0a66d9f0a10bec0efa1883ca75ee9e55baf511909d6"}, + {file = "pydantic-1.10.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a9a6747cac06c2beb466064dda999a13176b23535e4c496c9d48e6406f92d42d"}, + {file = "pydantic-1.10.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb992a1ef739cc7b543576337bebfc62c0e6567434e522e97291b251a41dad7f"}, + {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:990406d226dea0e8f25f643b370224771878142155b879784ce89f633541a024"}, + {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e82a6d37a95e0b1b42b82ab340ada3963aea1317fd7f888bb6b9dfbf4fff57c"}, + {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9193d4f4ee8feca58bc56c8306bcb820f5c7905fd919e0750acdeeeef0615b28"}, + {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2b3ce5f16deb45c472dde1a0ee05619298c864a20cded09c4edd820e1454129f"}, + {file = "pydantic-1.10.4-cp39-cp39-win_amd64.whl", hash = "sha256:9cbdc268a62d9a98c56e2452d6c41c0263d64a2009aac69246486f01b4f594c4"}, + {file = "pydantic-1.10.4-py3-none-any.whl", hash = "sha256:4948f264678c703f3877d1c8877c4e3b2e12e549c57795107f08cf70c6ec7774"}, + {file = "pydantic-1.10.4.tar.gz", hash = "sha256:b9a3859f24eb4e097502a3be1fb4b2abb79b6103dd9e2e0edb70613a4459a648"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pydevlake" +version = "0.1.0" +description = "Devlake plugin framework" +category = "main" +optional = false +python-versions = "^3.10" +files = [] +develop = false + +[package.dependencies] +fire = "^0.4.0" +inflect = "^6.0.2" +mysqlclient = "^2.1.1" +pydantic = "^1.10.2" +requests = "^2.28.1" +sqlmodel = "^0.0.8" + +[package.source] +type = "directory" +url = "../../../python/pydevlake" + +[[package]] +name = "requests" +version = "2.28.2" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7, <4" +files = [ + {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, + {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "sqlalchemy" +version = "1.4.41" +description = "Database Abstraction Library" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "SQLAlchemy-1.4.41-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:13e397a9371ecd25573a7b90bd037db604331cf403f5318038c46ee44908c44d"}, + {file = "SQLAlchemy-1.4.41-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2d6495f84c4fd11584f34e62f9feec81bf373787b3942270487074e35cbe5330"}, + {file = "SQLAlchemy-1.4.41-cp27-cp27m-win32.whl", hash = "sha256:e570cfc40a29d6ad46c9aeaddbdcee687880940a3a327f2c668dd0e4ef0a441d"}, + {file = "SQLAlchemy-1.4.41-cp27-cp27m-win_amd64.whl", hash = "sha256:5facb7fd6fa8a7353bbe88b95695e555338fb038ad19ceb29c82d94f62775a05"}, + {file = "SQLAlchemy-1.4.41-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:f37fa70d95658763254941ddd30ecb23fc4ec0c5a788a7c21034fc2305dab7cc"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:361f6b5e3f659e3c56ea3518cf85fbdae1b9e788ade0219a67eeaaea8a4e4d2a"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0990932f7cca97fece8017414f57fdd80db506a045869d7ddf2dda1d7cf69ecc"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cd767cf5d7252b1c88fcfb58426a32d7bd14a7e4942497e15b68ff5d822b41ad"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5102fb9ee2c258a2218281adcb3e1918b793c51d6c2b4666ce38c35101bb940e"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-win32.whl", hash = "sha256:2082a2d2fca363a3ce21cfa3d068c5a1ce4bf720cf6497fb3a9fc643a8ee4ddd"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-win_amd64.whl", hash = "sha256:e4b12e3d88a8fffd0b4ca559f6d4957ed91bd4c0613a4e13846ab8729dc5c251"}, + {file = "SQLAlchemy-1.4.41-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:90484a2b00baedad361402c257895b13faa3f01780f18f4a104a2f5c413e4536"}, + {file = "SQLAlchemy-1.4.41-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b67fc780cfe2b306180e56daaa411dd3186bf979d50a6a7c2a5b5036575cbdbb"}, + {file = "SQLAlchemy-1.4.41-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ad2b727fc41c7f8757098903f85fafb4bf587ca6605f82d9bf5604bd9c7cded"}, + {file = "SQLAlchemy-1.4.41-cp311-cp311-win32.whl", hash = "sha256:59bdc291165b6119fc6cdbc287c36f7f2859e6051dd923bdf47b4c55fd2f8bd0"}, + {file = "SQLAlchemy-1.4.41-cp311-cp311-win_amd64.whl", hash = "sha256:d2e054aed4645f9b755db85bc69fc4ed2c9020c19c8027976f66576b906a74f1"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:4ba7e122510bbc07258dc42be6ed45997efdf38129bde3e3f12649be70683546"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0dcf127bb99458a9d211e6e1f0f3edb96c874dd12f2503d4d8e4f1fd103790b"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e16c2be5cb19e2c08da7bd3a87fed2a0d4e90065ee553a940c4fc1a0fb1ab72b"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5ebeeec5c14533221eb30bad716bc1fd32f509196318fb9caa7002c4a364e4c"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-win32.whl", hash = "sha256:3e2ef592ac3693c65210f8b53d0edcf9f4405925adcfc031ff495e8d18169682"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-win_amd64.whl", hash = "sha256:eb30cf008850c0a26b72bd1b9be6730830165ce049d239cfdccd906f2685f892"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:c23d64a0b28fc78c96289ffbd0d9d1abd48d267269b27f2d34e430ea73ce4b26"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8eb8897367a21b578b26f5713833836f886817ee2ffba1177d446fa3f77e67c8"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:14576238a5f89bcf504c5f0a388d0ca78df61fb42cb2af0efe239dc965d4f5c9"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:639e1ae8d48b3c86ffe59c0daa9a02e2bfe17ca3d2b41611b30a0073937d4497"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-win32.whl", hash = "sha256:0005bd73026cd239fc1e8ccdf54db58b6193be9a02b3f0c5983808f84862c767"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-win_amd64.whl", hash = "sha256:5323252be2bd261e0aa3f33cb3a64c45d76829989fa3ce90652838397d84197d"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:05f0de3a1dc3810a776275763764bb0015a02ae0f698a794646ebc5fb06fad33"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0002e829142b2af00b4eaa26c51728f3ea68235f232a2e72a9508a3116bd6ed0"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:22ff16cedab5b16a0db79f1bc99e46a6ddececb60c396562e50aab58ddb2871c"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccfd238f766a5bb5ee5545a62dd03f316ac67966a6a658efb63eeff8158a4bbf"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-win32.whl", hash = "sha256:58bb65b3274b0c8a02cea9f91d6f44d0da79abc993b33bdedbfec98c8440175a"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-win_amd64.whl", hash = "sha256:ce8feaa52c1640de9541eeaaa8b5fb632d9d66249c947bb0d89dd01f87c7c288"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:199a73c31ac8ea59937cc0bf3dfc04392e81afe2ec8a74f26f489d268867846c"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4676d51c9f6f6226ae8f26dc83ec291c088fe7633269757d333978df78d931ab"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:036d8472356e1d5f096c5e0e1a7e0f9182140ada3602f8fff6b7329e9e7cfbcd"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2307495d9e0ea00d0c726be97a5b96615035854972cc538f6e7eaed23a35886c"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-win32.whl", hash = "sha256:9c56e19780cd1344fcd362fd6265a15f48aa8d365996a37fab1495cae8fcd97d"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-win_amd64.whl", hash = "sha256:f5fa526d027d804b1f85cdda1eb091f70bde6fb7d87892f6dd5a48925bc88898"}, + {file = "SQLAlchemy-1.4.41.tar.gz", hash = "sha256:0292f70d1797e3c54e862e6f30ae474014648bc9c723e14a2fda730adb0a9791"}, +] + +[package.dependencies] +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} + +[package.extras] +aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] +asyncio = ["greenlet (!=0.4.17)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"] +mssql = ["pyodbc"] +mssql-pymssql = ["pymssql"] +mssql-pyodbc = ["pyodbc"] +mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] +mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] +mysql-connector = ["mysql-connector-python"] +oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"] +postgresql = ["psycopg2 (>=2.7)"] +postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] +postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] +postgresql-psycopg2binary = ["psycopg2-binary"] +postgresql-psycopg2cffi = ["psycopg2cffi"] +pymysql = ["pymysql", "pymysql (<1)"] +sqlcipher = ["sqlcipher3_binary"] + +[[package]] +name = "sqlalchemy2-stubs" +version = "0.0.2a32" +description = "Typing Stubs for SQLAlchemy 1.4" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "sqlalchemy2-stubs-0.0.2a32.tar.gz", hash = "sha256:2a2cfab71d35ac63bf21ad841d8610cd93a3bd4c6562848c538fa975585c2739"}, + {file = "sqlalchemy2_stubs-0.0.2a32-py3-none-any.whl", hash = "sha256:7f5fb30b0cf7c6b74c50c1d94df77ff32007afee8d80499752eb3fedffdbdfb8"}, +] + +[package.dependencies] +typing-extensions = ">=3.7.4" + +[[package]] +name = "sqlmodel" +version = "0.0.8" +description = "SQLModel, SQL databases in Python, designed for simplicity, compatibility, and robustness." +category = "main" +optional = false +python-versions = ">=3.6.1,<4.0.0" +files = [ + {file = "sqlmodel-0.0.8-py3-none-any.whl", hash = "sha256:0fd805719e0c5d4f22be32eb3ffc856eca3f7f20e8c7aa3e117ad91684b518ee"}, + {file = "sqlmodel-0.0.8.tar.gz", hash = "sha256:3371b4d1ad59d2ffd0c530582c2140b6c06b090b32af9b9c6412986d7b117036"}, +] + +[package.dependencies] +pydantic = ">=1.8.2,<2.0.0" +SQLAlchemy = ">=1.4.17,<=1.4.41" +sqlalchemy2-stubs = "*" + +[[package]] +name = "termcolor" +version = "2.2.0" +description = "ANSI color formatting for output in terminal" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "termcolor-2.2.0-py3-none-any.whl", hash = "sha256:91ddd848e7251200eac969846cbae2dacd7d71c2871e92733289e7e3666f48e7"}, + {file = "termcolor-2.2.0.tar.gz", hash = "sha256:dfc8ac3f350788f23b2947b3e6cfa5a53b630b612e6cd8965a015a776020b99a"}, +] + +[package.extras] +tests = ["pytest", "pytest-cov"] + +[[package]] +name = "typing-extensions" +version = "4.4.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, +] + +[[package]] +name = "urllib3" +version = "1.26.14" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, + {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "ceb16988b541afeca0e4c6ecf1099ad7bb1c58ddafbf3884455c734e6494c283" diff --git a/backend/test/remote/fakeplugin/pyproject.toml b/backend/test/remote/fakeplugin/pyproject.toml new file mode 100644 index 00000000000..c91fa4692ec --- /dev/null +++ b/backend/test/remote/fakeplugin/pyproject.toml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[tool.poetry] +name = "fakeplugin" +version = "0.1.0" +description = "Fake python plugin used only in tests" +authors = [] + +[tool.poetry.dependencies] +python = "^3.10" +pydevlake = { path = "../../../python/pydevlake", develop = false } + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/backend/test/remote/fakeplugin/start.sh b/backend/test/remote/fakeplugin/start.sh new file mode 100755 index 00000000000..6a9d005eac4 --- /dev/null +++ b/backend/test/remote/fakeplugin/start.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +endpoint="$1" + +cd "${0%/*}" # make sure we're in the correct dir + +echo "Registering fake python plugin" &&\ +poetry run python fakeplugin/main.py startup "$endpoint" &&\ +exit_code=$? +if [ $exit_code != 0 ]; then + exit $exit_code +fi diff --git a/backend/test/remote/remote_test.go b/backend/test/remote/remote_test.go new file mode 100644 index 00000000000..4ecbc8768d1 --- /dev/null +++ b/backend/test/remote/remote_test.go @@ -0,0 +1,113 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "fmt" + "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/test/helper" + "github.com/stretchr/testify/require" + "os" + "os/exec" + "path/filepath" + "testing" + "time" +) + +const PLUGIN_NAME = "fake" + +func setupEnv() { + fmt.Println("Setup test env") + helper.LocalInit() + _ = os.Setenv("REMOTE_PLUGINS_STARTUP_PATH", filepath.Join("backend/test/remote/fakeplugin/start.sh")) + _ = os.Setenv("ENABLE_REMOTE_PLUGINS", "true") +} + +func buildPython(t *testing.T) { + fmt.Println("Build fake plugin") + path := filepath.Join(helper.ProjectRoot, "backend/test/remote/fakeplugin/build.sh") + cmd := exec.Command(helper.Shell, []string{path}...) + cmd.Dir = filepath.Dir(path) + cmd.Env = append(cmd.Env, os.Environ()...) + r, err := utils.RunProcess(cmd, + &utils.RunProcessOptions{ + OnStdout: func(b []byte) { + fmt.Println(string(b)) + }, + OnStderr: func(b []byte) { + fmt.Println(string(b)) + }, + }) + require.NoError(t, err) + require.NoError(t, r.GetError()) +} + +func connectLocalServer(t *testing.T) *helper.DevlakeClient { + fmt.Println("Connect to server") + client := helper.ConnectLocalServer(t, &helper.LocalClientConfig{ + ServerPort: 8089, + DbURL: helper.UseMySQL("127.0.0.1", 3307), + CreateServer: true, + DropDb: true, + Plugins: map[string]plugin.PluginMeta{}, + }) + client.SetTimeout(60 * time.Second) + // Wait for plugin registration + time.Sleep(3 * time.Second) + return client +} + +func TestRunPipeline(t *testing.T) { + setupEnv() + buildPython(t) + client := connectLocalServer(t) + fmt.Println("Create new connection") + conn := client.CreateConnection(PLUGIN_NAME, + api.AccessToken{ + Token: "this_is_a_valid_token", + }, + ) + client.SetTimeout(0) + conns := client.ListConnections(PLUGIN_NAME) + require.Equal(t, 1, len(conns)) + require.Equal(t, "this_is_a_valid_token", conns[0].Token) + fmt.Println("Run pipeline") + t.Run("run_pipeline", func(t *testing.T) { + pipeline := client.RunPipeline(models.NewPipeline{ + Name: "remote_test", + Plan: []plugin.PipelineStage{ + { + { + Plugin: PLUGIN_NAME, + Subtasks: nil, + Options: map[string]interface{}{ + "connectionId": conn.ID, + "scopeId": 1, + }, + }, + }, + }, + }) + require.Equal(t, models.TASK_COMPLETED, pipeline.Status) + require.Equal(t, 1, pipeline.FinishedTasks) + require.Equal(t, "", pipeline.ErrorName) + }) +}