diff --git a/backend/plugins/jira/e2e/migration_schema_test.go b/backend/plugins/jira/e2e/migration_schema_test.go new file mode 100644 index 00000000000..08124fa6459 --- /dev/null +++ b/backend/plugins/jira/e2e/migration_schema_test.go @@ -0,0 +1,112 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "sync" + "testing" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/migration" + coreMigration "github.com/apache/incubator-devlake/core/models/migrationscripts" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/impls/dalgorm" + "github.com/apache/incubator-devlake/plugins/jira/impl" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm/schema" +) + +// TestMigrationSchema guards against schema drift between Jira's migration +// scripts and its runtime GORM models. +// +// Regression test for the Sprint Report bug (introduced by PR #8967/#9010): +// the migration that created `_tool_jira_sprint_reports` used a struct that did +// NOT embed common.NoPKModel, so the `_raw_data_table` / `_raw_data_params` / +// `_raw_data_id` / `_raw_data_remark` columns were missing. The runtime model +// DID embed common.NoPKModel, so the ApiExtractor's cleanup query +// (`WHERE _raw_data_table = ? AND _raw_data_params = ?`) failed at runtime with +// "Error 1054: Unknown column '_raw_data_table' in 'where clause'". +// +// The test runs the REAL migration scripts (framework + jira) to build the +// schema exactly the way a production install would — deliberately NOT via +// AutoMigrate on the runtime model, which would silently hide such drift — and +// then asserts that every column each runtime model expects actually exists in +// the migrated table. Any future migration that forgets to embed +// common.NoPKModel (or otherwise omits a column) will fail this test. +// +// Requires E2E_DB_URL (runs under `make e2e-test` / `make e2e-test-go-plugins`). +func TestMigrationSchema(t *testing.T) { + var pluginInstance impl.Jira + dataflowTester := e2ehelper.NewDataFlowTester(t, "jira", pluginInstance) + + // Some jira migration scripts refuse to run without an encryption secret + // (20220716: "jira v0.11 invalid encKey"); CI does not necessarily provide + // one, so fall back to a deterministic test value. dalgorm.Init registers + // the `encdec` GORM serializer used by connection models — without it the + // migrations fail with "invalid serializer type encdec". + if dataflowTester.Cfg.GetString(plugin.EncodeKeyEnvStr) == "" { + dataflowTester.Cfg.Set(plugin.EncodeKeyEnvStr, "jira-e2e-test-encryption-secret") + } + dalgorm.Init(dataflowTester.Cfg.GetString(plugin.EncodeKeyEnvStr)) + + // Apply the real migration scripts so the schema matches a production install. + basicRes := runner.CreateBasicRes(dataflowTester.Cfg, dataflowTester.Log, dataflowTester.Db) + migrator, err := migration.NewMigrator(basicRes) + require.NoError(t, err) + migrator.Register(coreMigration.All(), "Framework") + migrator.Register(pluginInstance.MigrationScripts(), "jira") + require.NoError(t, migrator.Execute()) + + keepAll := func(dal.ColumnMeta) bool { return true } + + for _, table := range pluginInstance.GetTablesInfo() { + table := table + t.Run(table.TableName(), func(t *testing.T) { + // Columns the runtime GORM model expects. + sch, err := schema.Parse(table, &sync.Map{}, schema.NamingStrategy{}) + require.NoErrorf(t, err, "unable to parse schema for %T", table) + + // Columns that actually exist in the migrated table. + actualColumns, err := dal.GetColumnNames(dataflowTester.Dal, table, keepAll) + if err != nil || len(actualColumns) == 0 { + // No migration creates this table (e.g. API response models + // that are listed in GetTablesInfo but never persisted) — + // there is no schema to drift from. + t.Skipf("table %q not present after migrations", table.TableName()) + } + existing := make(map[string]struct{}, len(actualColumns)) + for _, c := range actualColumns { + existing[c] = struct{}{} + } + + for _, field := range sch.Fields { + if field.DBName == "" || field.IgnoreMigration { + continue + } + _, ok := existing[field.DBName] + assert.Truef(t, ok, + "table %q is missing column %q expected by model %T — "+ + "did a migration script forget to embed common.NoPKModel (raw-data columns) or add the field?", + table.TableName(), field.DBName, table) + } + }) + } +} diff --git a/backend/plugins/jira/models/migrationscripts/20260727_add_raw_data_columns_to_sprint_report.go b/backend/plugins/jira/models/migrationscripts/20260727_add_raw_data_columns_to_sprint_report.go new file mode 100644 index 00000000000..5ee4d671f70 --- /dev/null +++ b/backend/plugins/jira/models/migrationscripts/20260727_add_raw_data_columns_to_sprint_report.go @@ -0,0 +1,65 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// jiraSprintReport20260727 mirrors the JiraSprintReport model. The original +// migration (20260722) created _tool_jira_sprint_reports without embedding +// common.NoPKModel, so the _raw_data_table / _raw_data_params / _raw_data_id / +// _raw_data_remark columns (and created_at / updated_at) were missing. The +// runtime model expects them, which made the ApiExtractor's cleanup query +// (WHERE _raw_data_table = ? AND _raw_data_params = ?) fail with +// "Unknown column '_raw_data_table' in 'where clause'". Re-running +// AutoMigrateTables adds the missing columns without dropping existing data. +type jiraSprintReport20260727 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + BoardId uint64 `gorm:"primaryKey"` + SprintId uint64 `gorm:"primaryKey"` + IssueId uint64 `gorm:"primaryKey"` + + IssueKey string `gorm:"type:varchar(255)"` + Bucket string `gorm:"type:varchar(32);index"` + Done bool + StoryPointsAtSprintStart *float64 + StoryPointsAtSprintEnd *float64 +} + +func (jiraSprintReport20260727) TableName() string { + return "_tool_jira_sprint_reports" +} + +type addRawDataColumnsToSprintReport struct{} + +func (script *addRawDataColumnsToSprintReport) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &jiraSprintReport20260727{}) +} + +func (*addRawDataColumnsToSprintReport) Version() uint64 { + return 20260727000000 +} + +func (*addRawDataColumnsToSprintReport) Name() string { + return "add missing _raw_data_* columns to _tool_jira_sprint_reports" +} diff --git a/backend/plugins/jira/models/migrationscripts/register.go b/backend/plugins/jira/models/migrationscripts/register.go index 90a3317bd14..e71dee8ea14 100644 --- a/backend/plugins/jira/models/migrationscripts/register.go +++ b/backend/plugins/jira/models/migrationscripts/register.go @@ -59,5 +59,6 @@ func All() []plugin.MigrationScript { new(changeFixVersionsToText20260707), new(addExtraJQLToScopeConfig), new(addSprintReportTable), + new(addRawDataColumnsToSprintReport), } } diff --git a/backend/plugins/schema_e2e/migration_schema_test.go b/backend/plugins/schema_e2e/migration_schema_test.go new file mode 100644 index 00000000000..43cb0834c99 --- /dev/null +++ b/backend/plugins/schema_e2e/migration_schema_test.go @@ -0,0 +1,250 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package schema_e2e contains a cross-plugin regression guard that runs the +// REAL migration scripts of every built-in Go plugin and then asserts that the +// resulting database schema still matches what each runtime GORM model expects. +// +// It lives in an `e2e` package on purpose: it needs a real database +// (E2E_DB_URL) and is therefore only executed by `make e2e-test-go-plugins` +// (scripts/e2e-test-go-plugins.sh selects packages whose import path contains +// "e2e"), and excluded from the DB-less unit test run +// (scripts/unit-test-go.sh skips paths matching "e2e"). +package schema_e2e + +import ( + "os" + "path/filepath" + "sync" + "testing" + + "github.com/apache/incubator-devlake/core/config" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/migration" + coreMigration "github.com/apache/incubator-devlake/core/models/migrationscripts" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/impls/dalgorm" + "github.com/apache/incubator-devlake/impls/logruslog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm/schema" + + ae "github.com/apache/incubator-devlake/plugins/ae/impl" + argocd "github.com/apache/incubator-devlake/plugins/argocd/impl" + asana "github.com/apache/incubator-devlake/plugins/asana/impl" + azuredevops "github.com/apache/incubator-devlake/plugins/azuredevops_go/impl" + bamboo "github.com/apache/incubator-devlake/plugins/bamboo/impl" + bitbucket "github.com/apache/incubator-devlake/plugins/bitbucket/impl" + bitbucket_server "github.com/apache/incubator-devlake/plugins/bitbucket_server/impl" + circleci "github.com/apache/incubator-devlake/plugins/circleci/impl" + claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" + customize "github.com/apache/incubator-devlake/plugins/customize/impl" + dbt "github.com/apache/incubator-devlake/plugins/dbt/impl" + dora "github.com/apache/incubator-devlake/plugins/dora/impl" + feishu "github.com/apache/incubator-devlake/plugins/feishu/impl" + copilot "github.com/apache/incubator-devlake/plugins/gh-copilot/impl" + gitee "github.com/apache/incubator-devlake/plugins/gitee/impl" + gitextractor "github.com/apache/incubator-devlake/plugins/gitextractor/impl" + github "github.com/apache/incubator-devlake/plugins/github/impl" + githubGraphql "github.com/apache/incubator-devlake/plugins/github_graphql/impl" + gitlab "github.com/apache/incubator-devlake/plugins/gitlab/impl" + icla "github.com/apache/incubator-devlake/plugins/icla/impl" + issueTrace "github.com/apache/incubator-devlake/plugins/issue_trace/impl" + jenkins "github.com/apache/incubator-devlake/plugins/jenkins/impl" + jira "github.com/apache/incubator-devlake/plugins/jira/impl" + linear "github.com/apache/incubator-devlake/plugins/linear/impl" + linker "github.com/apache/incubator-devlake/plugins/linker/impl" + opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" + org "github.com/apache/incubator-devlake/plugins/org/impl" + pagerduty "github.com/apache/incubator-devlake/plugins/pagerduty/impl" + q_dev "github.com/apache/incubator-devlake/plugins/q_dev/impl" + refdiff "github.com/apache/incubator-devlake/plugins/refdiff/impl" + rootly "github.com/apache/incubator-devlake/plugins/rootly/impl" + slack "github.com/apache/incubator-devlake/plugins/slack/impl" + sonarqube "github.com/apache/incubator-devlake/plugins/sonarqube/impl" + starrocks "github.com/apache/incubator-devlake/plugins/starrocks/impl" + taiga "github.com/apache/incubator-devlake/plugins/taiga/impl" + tapd "github.com/apache/incubator-devlake/plugins/tapd/impl" + teambition "github.com/apache/incubator-devlake/plugins/teambition/impl" + tempo "github.com/apache/incubator-devlake/plugins/tempo/impl" + testmo "github.com/apache/incubator-devlake/plugins/testmo/impl" + trello "github.com/apache/incubator-devlake/plugins/trello/impl" + webhook "github.com/apache/incubator-devlake/plugins/webhook/impl" + zentao "github.com/apache/incubator-devlake/plugins/zentao/impl" +) + +// allGoPlugins lists EVERY built-in Go plugin. Keep it in sync with the plugin +// directories under backend/plugins/ (the TestAllGoPluginsListed guard below +// fails if a new plugin's `impl` package is added but not registered here). +func allGoPlugins() []plugin.PluginMeta { + return []plugin.PluginMeta{ + ae.AE{}, + argocd.ArgoCD{}, + asana.Asana{}, + azuredevops.Azuredevops{}, + bamboo.Bamboo{}, + bitbucket.Bitbucket{}, + bitbucket_server.BitbucketServer{}, + circleci.Circleci{}, + claudeCode.ClaudeCode{}, + customize.Customize{}, + dbt.Dbt{}, + dora.Dora{}, + feishu.Feishu{}, + copilot.GhCopilot{}, + gitee.Gitee{}, + gitextractor.GitExtractor{}, + github.Github{}, + githubGraphql.GithubGraphql{}, + gitlab.Gitlab{}, + icla.Icla{}, + issueTrace.IssueTrace{}, + jenkins.Jenkins{}, + jira.Jira{}, + linear.Linear{}, + linker.Linker{}, + opsgenie.Opsgenie{}, + org.Org{}, + pagerduty.PagerDuty{}, + q_dev.QDev{}, + refdiff.RefDiff{}, + rootly.Rootly{}, + slack.Slack{}, + sonarqube.Sonarqube{}, + starrocks.StarRocks{}, + taiga.Taiga{}, + tapd.Tapd{}, + teambition.Teambition{}, + tempo.Tempo{}, + testmo.Testmo{}, + trello.Trello{}, + webhook.Webhook{}, + zentao.Zentao{}, + } +} + +// TestAllGoPluginsListed guarantees allGoPlugins() stays complete: it counts the +// plugin directories that ship an `impl` package and fails if that number does +// not match the registered list. This makes the schema-drift guard below +// automatically cover any newly added plugin. +func TestAllGoPluginsListed(t *testing.T) { + entries, err := os.ReadDir("..") + require.NoError(t, err) + dirsWithImpl := 0 + for _, e := range entries { + if !e.IsDir() { + continue + } + if info, statErr := os.Stat(filepath.Join("..", e.Name(), "impl")); statErr == nil && info.IsDir() { + dirsWithImpl++ + } + } + assert.Equalf(t, dirsWithImpl, len(allGoPlugins()), + "number of plugin dirs with an impl/ package (%d) != registered plugins (%d); "+ + "add the new plugin to allGoPlugins() in plugins/schema_e2e/migration_schema_test.go", + dirsWithImpl, len(allGoPlugins())) +} + +// TestMigrationSchemaMatchesModels applies the real framework + plugin migration +// scripts and then verifies, for every plugin model, that each column the +// runtime GORM model declares actually exists in the migrated table. +// +// This is a cross-plugin generalization of the Jira Sprint Report regression: +// a migration created `_tool_jira_sprint_reports` without embedding +// common.NoPKModel, so the `_raw_data_*` columns were missing and the +// ApiExtractor cleanup query failed at runtime with +// "Unknown column '_raw_data_table' in 'where clause'". +// +// Tables that no migration creates (e.g. models materialized lazily at runtime) +// are skipped, so the check specifically targets *drift* between an existing +// table and its model — which is exactly the failure mode above. +func TestMigrationSchemaMatchesModels(t *testing.T) { + cfg := config.GetConfig() + e2eDbURL := cfg.GetString("E2E_DB_URL") + if e2eDbURL == "" { + t.Skip("E2E_DB_URL is not set; skipping cross-plugin migration schema check") + } + cfg.Set("DB_URL", e2eDbURL) + // Some migration scripts refuse to run without an encryption secret + // (e.g. jira 20220716: "jira v0.11 invalid encKey"). CI does not + // necessarily provide one, so fall back to a deterministic test value. + if cfg.GetString(plugin.EncodeKeyEnvStr) == "" { + cfg.Set(plugin.EncodeKeyEnvStr, "schema-e2e-test-encryption-secret") + } + + db, err := runner.NewGormDb(cfg, logruslog.Global) + require.NoError(t, err) + // register the `encdec` GORM serializer, otherwise migration scripts that + // touch connection models fail with "invalid serializer type encdec" + // (runner.CreateBasicRes does not do this, only CreateAppBasicRes does). + dalgorm.Init(cfg.GetString(plugin.EncodeKeyEnvStr)) + dalInstance := dalgorm.NewDalgorm(db) + basicRes := runner.CreateBasicRes(cfg, logruslog.Global, db) + + // Apply the migrations exactly the way the server does on startup. + migrator, migErr := migration.NewMigrator(basicRes) + require.NoError(t, migErr) + migrator.Register(coreMigration.All(), "Framework") + for _, p := range allGoPlugins() { + if migratable, ok := p.(plugin.PluginMigration); ok { + migrator.Register(migratable.MigrationScripts(), p.Name()) + } + } + require.NoError(t, migrator.Execute()) + + keepAll := func(dal.ColumnMeta) bool { return true } + + for _, p := range allGoPlugins() { + modeler, ok := p.(plugin.PluginModel) + if !ok { + continue + } + p := p + t.Run(p.Name(), func(t *testing.T) { + for _, table := range modeler.GetTablesInfo() { + table := table + // Columns that actually exist in the migrated table. + actualColumns, colErr := dal.GetColumnNames(dalInstance, table, keepAll) + if colErr != nil || len(actualColumns) == 0 { + // No migration created this table (e.g. runtime-only / + // dynamic model) — nothing to validate for drift. + t.Logf("skip %q: table not present after migrations", table.TableName()) + continue + } + existing := make(map[string]struct{}, len(actualColumns)) + for _, c := range actualColumns { + existing[c] = struct{}{} + } + + // Columns the runtime GORM model expects. + sch, parseErr := schema.Parse(table, &sync.Map{}, schema.NamingStrategy{}) + require.NoErrorf(t, parseErr, "unable to parse schema for %T", table) + for _, field := range sch.Fields { + if field.DBName == "" || field.IgnoreMigration { + continue + } + _, present := existing[field.DBName] + assert.Truef(t, present, + "[%s] table %q is missing column %q expected by model %T — "+ + "did a migration script forget to embed common.NoPKModel (raw-data columns) or add the field?", + p.Name(), table.TableName(), field.DBName, table) + } + } + }) + } +} diff --git a/backend/plugins/taiga/models/migrationscripts/20260727_add_missing_scope_config_columns.go b/backend/plugins/taiga/models/migrationscripts/20260727_add_missing_scope_config_columns.go new file mode 100644 index 00000000000..84fa66cda41 --- /dev/null +++ b/backend/plugins/taiga/models/migrationscripts/20260727_add_missing_scope_config_columns.go @@ -0,0 +1,61 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// taigaScopeConfig20260727 mirrors models.TaigaScopeConfig. The initial +// migration created `_tool_taiga_scope_configs` without the `type_mappings` +// column, while the runtime model declares it — every read/write of the model +// would fail with "Unknown column 'type_mappings'". +// +// The `uniqueIndex` on `name` is safe to add here: the new column is nullable, +// so pre-existing rows are backfilled with NULL, and both MySQL and PostgreSQL +// allow duplicate NULLs in a unique index (verified against both engines). +type taigaScopeConfig20260727 struct { + archived.Model + Entities []string `gorm:"type:json;serializer:json" json:"entities"` + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `json:"name" gorm:"type:varchar(255);uniqueIndex"` + TypeMappings map[string]json.RawMessage `json:"typeMappings" gorm:"type:json;serializer:json"` +} + +func (taigaScopeConfig20260727) TableName() string { + return "_tool_taiga_scope_configs" +} + +type addMissingScopeConfigColumns struct{} + +func (script *addMissingScopeConfigColumns) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &taigaScopeConfig20260727{}) +} + +func (*addMissingScopeConfigColumns) Version() uint64 { + return 20260727000001 +} + +func (*addMissingScopeConfigColumns) Name() string { + return "add missing type_mappings column to _tool_taiga_scope_configs" +} diff --git a/backend/plugins/taiga/models/migrationscripts/register.go b/backend/plugins/taiga/models/migrationscripts/register.go index d2cfd08e269..da91884aa1f 100644 --- a/backend/plugins/taiga/models/migrationscripts/register.go +++ b/backend/plugins/taiga/models/migrationscripts/register.go @@ -26,5 +26,6 @@ func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(addInitTables20250220), new(addTaskIssueEpicTables20260306), + new(addMissingScopeConfigColumns), } } diff --git a/backend/plugins/teambition/models/migrationscripts/20260727_add_missing_scope_config_columns.go b/backend/plugins/teambition/models/migrationscripts/20260727_add_missing_scope_config_columns.go new file mode 100644 index 00000000000..fc2addaf076 --- /dev/null +++ b/backend/plugins/teambition/models/migrationscripts/20260727_add_missing_scope_config_columns.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// teambitionScopeConfig20260727 mirrors models.TeambitionScopeConfig. The +// migration that created `_tool_teambition_scope_configs` did not include the +// columns of the embedded common.Model (`id`, `created_at`, `updated_at`), +// which the runtime model expects. +// +// Adding the AUTO_INCREMENT primary key `id` to the already existing (and +// primary-key-less) table works: GORM issues a plain ADD COLUMN and MySQL +// backfills consecutive ids for existing rows — verified against MySQL 8.4. +// The `uniqueIndex` on `name` is likewise safe, because newly added columns are +// nullable and both MySQL and PostgreSQL permit duplicate NULLs in a unique +// index. +type teambitionScopeConfig20260727 struct { + archived.Model + Entities []string `gorm:"type:json;serializer:json" json:"entities"` + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `json:"name" gorm:"type:varchar(255);uniqueIndex"` + TypeMappings map[string]string `json:"typeMappings" gorm:"serializer:json"` + StatusMappings map[string]string `json:"statusMappings" gorm:"serializer:json"` + BugDueDateField string `json:"bugDueDateField" gorm:"column:bug_due_date_field"` + TaskDueDateField string `json:"taskDueDateField" gorm:"column:task_due_date_field"` + StoryDueDateField string `json:"storyDueDateField" gorm:"column:story_due_date_field"` +} + +func (teambitionScopeConfig20260727) TableName() string { + return "_tool_teambition_scope_configs" +} + +type addMissingScopeConfigColumns struct{} + +func (script *addMissingScopeConfigColumns) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &teambitionScopeConfig20260727{}) +} + +func (*addMissingScopeConfigColumns) Version() uint64 { + return 20260727000001 +} + +func (*addMissingScopeConfigColumns) Name() string { + return "add missing id/created_at/updated_at columns to _tool_teambition_scope_configs" +} diff --git a/backend/plugins/teambition/models/migrationscripts/register.go b/backend/plugins/teambition/models/migrationscripts/register.go index f9914e7a12c..d761a15fb70 100644 --- a/backend/plugins/teambition/models/migrationscripts/register.go +++ b/backend/plugins/teambition/models/migrationscripts/register.go @@ -26,5 +26,6 @@ func All() []plugin.MigrationScript { new(reCreateTeambitionConnections), new(addScopeConfigId), new(addAppIdBack), + new(addMissingScopeConfigColumns), } } diff --git a/backend/plugins/testmo/models/migrationscripts/20260727_add_missing_scope_config_columns.go b/backend/plugins/testmo/models/migrationscripts/20260727_add_missing_scope_config_columns.go new file mode 100644 index 00000000000..11048f60a74 --- /dev/null +++ b/backend/plugins/testmo/models/migrationscripts/20260727_add_missing_scope_config_columns.go @@ -0,0 +1,61 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// testmoScopeConfig20260727 mirrors models.TestmoScopeConfig. The migration +// that created `_tool_testmo_scope_configs` omitted the `connection_id` and +// `name` columns of the embedded common.ScopeConfig, which the runtime model +// expects. +// +// The `uniqueIndex` on `name` is safe to add here: the new column is nullable, +// so pre-existing rows are backfilled with NULL, and both MySQL and PostgreSQL +// allow duplicate NULLs in a unique index (verified against both engines). +type testmoScopeConfig20260727 struct { + archived.Model + Entities []string `gorm:"type:json;serializer:json" json:"entities"` + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `json:"name" gorm:"type:varchar(255);uniqueIndex"` + AcceptanceTestPattern string `json:"acceptanceTestPattern" gorm:"type:varchar(255)"` + SmokeTestPattern string `json:"smokeTestPattern" gorm:"type:varchar(255)"` + TeamPattern string `json:"teamPattern" gorm:"type:varchar(255)"` +} + +func (testmoScopeConfig20260727) TableName() string { + return "_tool_testmo_scope_configs" +} + +type addMissingScopeConfigColumns struct{} + +func (script *addMissingScopeConfigColumns) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &testmoScopeConfig20260727{}) +} + +func (*addMissingScopeConfigColumns) Version() uint64 { + return 20260727000001 +} + +func (*addMissingScopeConfigColumns) Name() string { + return "add missing connection_id/name columns to _tool_testmo_scope_configs" +} diff --git a/backend/plugins/testmo/models/migrationscripts/register.go b/backend/plugins/testmo/models/migrationscripts/register.go index 7843d4b84e1..95f2ae48b41 100644 --- a/backend/plugins/testmo/models/migrationscripts/register.go +++ b/backend/plugins/testmo/models/migrationscripts/register.go @@ -25,5 +25,6 @@ func All() []plugin.MigrationScript { new(addScopeConfigIdToProjects), new(replaceTestsWithRuns), new(fixRawTableNamesAndSchemas), + new(addMissingScopeConfigColumns), } } diff --git a/backend/scripts/build-plugins.sh b/backend/scripts/build-plugins.sh index 03fb7b4f9fb..e0ef94bb9b7 100755 --- a/backend/scripts/build-plugins.sh +++ b/backend/scripts/build-plugins.sh @@ -52,7 +52,8 @@ fi if [ -z "$DEVLAKE_PLUGINS" ]; then echo "Building all plugins" - PLUGINS=$(find $PLUGIN_SRC_DIR/* -maxdepth 0 -type d -not -name core -not -name helper -not -name logs -not -empty) + # schema_e2e is not a plugin, it only holds the cross-plugin schema-drift e2e test + PLUGINS=$(find $PLUGIN_SRC_DIR/* -maxdepth 0 -type d -not -name core -not -name helper -not -name logs -not -name schema_e2e -not -empty) else echo "Building the following plugins: $PLUGIN" PLUGINS=