Skip to content

feat(pyplugins): tx rule scope ap is py #4438

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hezyin merged 5 commits into from
Feb 17, 2023
Merged

feat(pyplugins): tx rule scope ap is py #4438

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
07287bd
docs: Document hook execution order
Feb 14, 2023
679eed5
fix: Add missing initialization of remote plugins
Feb 17, 2023
40e3a2f
fix: Fix swagger body parameter for test connection endpoint
Feb 17, 2023
a95b221
feat: Implement txRule and scope plugin APIs
Feb 16, 2023
0f387b2
feat: Pass scopeId and transformation rule to python plugin
Feb 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions backend/core/runner/loader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,25 @@ package runner

import (
"fmt"
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"io/fs"
"path/filepath"
goplugin "plugin"
"strconv"
"strings"

"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/server/services/remote"
)

// LoadPlugins load plugins from local directory
func LoadPlugins(basicRes context.BasicRes) errors.Error {
remote_plugins_enabled, err := strconv.ParseBool(basicRes.GetConfig("ENABLE_REMOTE_PLUGINS"))
if err == nil && remote_plugins_enabled {
remote.Init(basicRes)
}

pluginsDir := basicRes.GetConfig("PLUGIN_DIR")
walkErr := filepath.WalkDir(pluginsDir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
Expand Down
3 changes: 3 additions & 0 deletions backend/python/pydevlake/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,9 @@ class MyAPI(API):

Here the method `authenticate` is a hook that is run on each request.
Similarly you can declare response hooks with `@response_hook`.
Multiple hooks are executed in the order of their declaration.
The `API` base class declares some hooks that are executed first.


#### Pagination

Expand Down
2 changes: 1 addition & 1 deletion backend/python/pydevlake/pydevlake/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

from .model import ToolModel
from .logger import logger
from .message import Connection
from .message import Connection, TransformationRule
from .plugin import Plugin
from .stream import Stream, Substream
from .context import Context
14 changes: 11 additions & 3 deletions backend/python/pydevlake/pydevlake/context.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,23 @@
from urllib.parse import urlparse, parse_qsl
from sqlmodel import SQLModel, create_engine

from pydevlake.message import Connection
from pydevlake.message import Connection, TransformationRule


class Context:
def __init__(self, db_url: str, connection_id: int, connection: Connection, options: dict):
def __init__(self,
db_url: str,
scope_id: str,
connection_id: int,
connection: Connection,
transformation_rule: TransformationRule = None,
options: dict = None):
self.db_url = db_url
self.scope_id = scope_id
self.connection_id = connection_id
self.connection = connection
self.options = options
self.transformation_rule = transformation_rule
self.options = options or {}
self._engine = None

@property
Expand Down
281 changes: 277 additions & 4 deletions backend/python/pydevlake/pydevlake/doc.template.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,13 +118,16 @@
"/plugins/$plugin_name/test": {
"post": {
"description": "Test if a connection is valid",
"body": {
"application/json": {
"parameters": [
{
"name": "connection",
"required": true,
"in": "body",
"schema": {
"$$ref": "#/components/schemas/connection"
}
}
}
]
},
"response": {
"200": {
Expand All @@ -134,11 +137,245 @@
"description": "The connection is not valid"
}
}
},
"/plugins/$plugin_name/connections/{connectionId}/scopes/{scopeId}": {
"get": {
"description": "Get a scope",
"parameters": [
{
"$$ref": "#/components/parameters/connectionId"
},
{
"$$ref": "#/components/parameters/scopeId"
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$$ref": "#/components/schemas/scope"
}
}
}
}
}
},
"patch": {
"description": "Update a scope",
"parameters": [
{
"$$ref": "#/components/parameters/connectionId"
},
{
"$$ref": "#/components/parameters/scopeId"
},
{
"name": "scope",
"required": true,
"in": "body",
"schema": {
"$$ref": "#/components/schemas/scope"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$$ref": "#/components/schemas/scope"
}
}
}
}
}
}
},
"/plugins/$plugin_name/connections/{connectionId}/scopes": {
"get": {
"description": "Get all scopes",
"parameters": [
{
"$$ref": "#/components/parameters/pageSize"
},
{
"$$ref": "#/components/parameters/page"
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$$ref": "#/components/schemas/scope"
}
}
}
}
}
}
},
"put": {
"description": "Create a list of scopes",
"parameters": [
{
"name": "scopes",
"required": true,
"in": "body",
"schema": {
"type": "array",
"items": {
"$$ref": "#/components/schemas/scope"
}
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$$ref": "#/components/schemas/scope"
}
}
}
}
}
}
}
},
"/plugins/$plugin_name/transformation_rules": {
"get": {
"description": "Get all transformation rules",
"parameters": [
{
"$$ref": "#/components/parameters/pageSize"
},
{
"$$ref": "#/components/parameters/page"
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$$ref": "#/components/schemas/transformationRule"
}
}
}
}
}
}
},
"post": {
"description": "Create a transformation rule",
"parameters": [
{
"name": "rules",
"required": true,
"in": "body",
"schema": {
"$$ref": "#/components/schemas/transformationRule"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$$ref": "#/components/schemas/transformationRule"
}
}
}
}
}
}
},
"/plugins/$plugin_name/transformation_rules/{ruleId}": {
"get": {
"description": "Get a transformation rule",
"parameters": [
{
"$$ref": "#/components/parameters/ruleId"
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$$ref": "#/components/schemas/transformationRule"
}
}
}
}
}
},
"patch": {
"description": "Update a transformation rule",
"parameters": [
{
"$$ref": "#/components/parameters/ruleId"
},
{
"name": "rule",
"required": true,
"in": "body",
"schema": {
"$$ref": "#/components/schemas/transformationRule"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$$ref": "#/components/schemas/transformationRule"
}
}
}
}
}
}
}
},
"components": {
"schemas": {
"connection": $connection_schema
"connection": $connection_schema,
"transformationRule": $transformation_rule_schema,
"scope": {
"title": "Scope",
"type": "object",
"properties": {
"scopeId": {
"title": "scope id",
"type": "string"
},
"scopeName": {
"title": "scope name",
"type": "string"
},
"connectionId": {
"title": "connection id",
"type": "integer"
},
"transformationRuleId": {
"title": "Transformation rule id",
"type": "integer"
}
},
"required": ["scopeId", "scopeName", "connectionId", "transformationRuleId"]
}
},
"parameters": {
"connectionId": {
Expand All @@ -149,6 +386,42 @@
"schema": {
"type": "int"
}
},
"scopeId": {
"name": "scopeId",
"description": "Id of the scope",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
},
"ruleId": {
"name": "ruleId",
"description": "Id of the transformation rule",
"in": "path",
"required": true,
"schema": {
"type": "int"
}
},
"pageSize": {
"name": "pageSize",
"description": "Number of items per page",
"in": "query",
"required": false,
"schema": {
"type": "integer"
}
},
"page": {
"name": "page",
"description": "Page number",
"in": "query",
"required": false,
"schema": {
"type": "integer"
}
}
}
}
Expand Down
Loading