[feat-3701][Framework]: Python framework support for plugins

.gitignore

@@ -159,3 +159,7 @@ backend/server/api/docs/swagger.json
 backend/server/api/docs/swagger.yaml
 backend/server/api/docs/docs.go
 *.result/
+
+# python
+*.pyc
+__pycache__

.licenserc.yaml

@@ -34,6 +34,7 @@ header:
     - '**/env.example'
     - '**/*.csv'
     - '**/*.json'
+    - '**/*.template.json'
     - '**/*.sql'
     - '**/*.svg'
     - '**/*.png'
@@ -52,6 +53,7 @@ header:
     - '**/.eslintrc'
     - 'deployment/helm/templates/_helpers.tpl'
     - '**/.nvmrc'
+    - '**/poetry.lock'
 
 
 

backend/Dockerfile

@@ -138,7 +138,9 @@ COPY python/requirements.txt /app/requirements.txt
 RUN python3 -m pip install --no-cache --upgrade pip setuptools && \
     python3 -m pip install --no-cache dbt-mysql dbt-postgres && \
     python3 -m pip install --no-cache -r requirements.txt && \
-    rm -fr /usr/share/python-wheels/*
+    rm -fr /usr/share/python-wheels/* \
+RUN curl -sSL https://install.python-poetry.org | python3 -
+RUN ln -sf /root/.local/bin/poetry /usr/local/bin
 
 
 FROM base as devlake-base

backend/core/config/config_viper.go

@@ -19,12 +19,13 @@ package config
 
 import (
 	"fmt"
-	"github.com/apache/incubator-devlake/core/errors"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 
+	"github.com/apache/incubator-devlake/core/errors"
+
 	goerror "github.com/cockroachdb/errors"
 
 	"github.com/sirupsen/logrus"
@@ -74,6 +75,8 @@ func setDefaultValue(v *viper.Viper) {
 	v.SetDefault("PLUGIN_DIR", "bin/plugins")
 	v.SetDefault("TEMPORAL_TASK_QUEUE", "DEVLAKE_TASK_QUEUE")
 	v.SetDefault("TAP_PROPERTIES_DIR", "resources/tap")
+	v.SetDefault("ENABLE_REMOTE_PLUGINS", "true")
+	v.SetDefault("REMOTE_PLUGINS_STARTUP_PATH", "python/plugins/start.sh")
 }
 
 // replaceNewEnvItemInOldContent replace old config to new config in env file content

backend/core/models/dynamic_tabler.go

@@ -0,0 +1,75 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+	"reflect"
+
+	"github.com/apache/incubator-devlake/core/dal"
+)
+
+// DynamicTabler is a core.Tabler that wraps a runtime (anonymously) generated data-model. Due to limitations of
+// reflection in Go and the GORM framework, the underlying model and the table have to be explicitly passed into dal.Dal's API
+// via Unwrap() and TableName()
+type DynamicTabler struct {
+	objType reflect.Type
+	wrapped any
+	table   string
+}
+
+func NewDynamicTabler(tableName string, objType reflect.Type) *DynamicTabler {
+	return &DynamicTabler{
+		objType: objType,
+		table:   tableName,
+	}
+}
+
+func (d *DynamicTabler) New() *DynamicTabler {
+	return &DynamicTabler{
+		objType: d.objType,
+		wrapped: reflect.New(d.objType).Interface(),
+		table:   d.table,
+	}
+}
+
+func (d *DynamicTabler) NewSlice() *DynamicTabler {
+	sliceType := reflect.SliceOf(d.objType)
+	return &DynamicTabler{
+		objType: sliceType,
+		wrapped: reflect.New(sliceType).Interface(),
+		table:   d.table,
+	}
+}
+
+func (d *DynamicTabler) Unwrap() any {
+	return d.wrapped
+}
+
+func (d *DynamicTabler) TableName() string {
+	return d.table
+}
+
+var _ dal.Tabler = (*DynamicTabler)(nil)
+
+// UnwrapObject if the actual object is wrapped in some proxy, it unwinds and returns it, otherwise this is idempotent
+func UnwrapObject(ifc any) any {
+	if dynamic, ok := ifc.(*DynamicTabler); ok {
+		return dynamic.Unwrap()
+	}
+	return ifc
+}