Skip to content

Commit

Permalink
Merge branch 'main' into incremental-metadata-extraction-config
Browse files Browse the repository at this point in the history
  • Loading branch information
chirag-madlani authored Jan 25, 2025
2 parents df0c87c + c817785 commit 1818a1f
Show file tree
Hide file tree
Showing 2,333 changed files with 50,453 additions and 12,762 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/auto-cherry-pick-labeled-prs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@ run-name: OpenMetadata release cherry-pick PR #${{ github.event.pull_request.num

# yamllint disable-line rule:truthy
on:
pull_request:
pull_request_target:
types: [closed]
branches:
- main
permissions:
contents: write
pull-requests: write
issues: write
env:
CURRENT_RELEASE_ENDPOINT: ${{ vars.CURRENT_RELEASE_ENDPOINT }} # Endpoint that returns the current release version in json format
jobs:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/playwright-integration-tests-mysql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ jobs:
source env/bin/activate
make install_e2e_tests
make run_e2e_tests
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
if: always()
with:
name: playwright-artifacts
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ jobs:
source env/bin/activate
make install_e2e_tests
make run_e2e_tests
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
if: always()
with:
name: playwright-artifacts
Expand Down
19 changes: 11 additions & 8 deletions .github/workflows/playwright-mysql-e2e-skip.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,21 @@ on:
- synchronize
- reopened
paths:
- openmetadata-docs/**
- .github/**
- openmetadata-dist/**
- docker/**
- 'openmetadata-docs/**'
- '.github/**'
- 'openmetadata-dist/**'
- 'docker/**'
- '!docker/development/docker-compose.yml'
- '!docker/development/docker-compose-postgres.yml'


jobs:
playwright-ci-mysql:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
shardIndex: [1, 2]
shardTotal: [2]
fail-fast: false
matrix:
shardIndex: [1, 2]
shardTotal: [2]
steps:
- run: 'echo "Step is not required"'
17 changes: 11 additions & 6 deletions .github/workflows/playwright-mysql-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,14 @@ on:
- synchronize
- reopened
paths-ignore:
- openmetadata-docs/**
- .github/**
- openmetadata-dist/**
- docker/**
- 'openmetadata-docs/**'
- '.github/**'
- 'openmetadata-dist/**'
- 'docker/**'
- '!docker/development/docker-compose.yml'
- '!docker/development/docker-compose-postgres.yml'


permissions:
contents: read

Expand All @@ -36,7 +39,7 @@ concurrency:

jobs:
playwright-ci-mysql:
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
environment: test
strategy:
fail-fast: false
Expand Down Expand Up @@ -97,7 +100,7 @@ jobs:
working-directory: openmetadata-ui/src/main/resources/ui/
run: yarn --ignore-scripts --frozen-lockfile
- name: Install Playwright Browsers
run: npx playwright@1.44.1 install --with-deps
run: npx playwright@1.48.2 install --with-deps
- name: Run Playwright tests
working-directory: openmetadata-ui/src/main/resources/ui/
run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
Expand Down Expand Up @@ -164,3 +167,5 @@ jobs:
cd ./docker/development
docker compose down --remove-orphans
sudo rm -rf ${PWD}/docker-volume
18 changes: 10 additions & 8 deletions .github/workflows/playwright-postgresql-e2e-skip.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,20 @@ on:
- synchronize
- reopened
paths:
- openmetadata-docs/**
- .github/**
- openmetadata-dist/**
- docker/**
- 'openmetadata-docs/**'
- '.github/**'
- 'openmetadata-dist/**'
- 'docker/**'
- '!docker/development/docker-compose.yml'
- '!docker/development/docker-compose-postgres.yml'

jobs:
playwright-ci-postgresql:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
shardIndex: [1, 2]
shardTotal: [2]
fail-fast: false
matrix:
shardIndex: [1, 2]
shardTotal: [2]
steps:
- run: 'echo "Step is not required"'
14 changes: 8 additions & 6 deletions .github/workflows/playwright-postgresql-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@ on:
- synchronize
- reopened
paths-ignore:
- openmetadata-docs/**
- .github/**
- openmetadata-dist/**
- docker/**
- 'openmetadata-docs/**'
- '.github/**'
- 'openmetadata-dist/**'
- 'docker/**'
- '!docker/development/docker-compose.yml'
- '!docker/development/docker-compose-postgres.yml'

permissions:
contents: read
Expand All @@ -36,7 +38,7 @@ concurrency:

jobs:
playwright-ci-postgresql:
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
environment: test
strategy:
fail-fast: false
Expand Down Expand Up @@ -97,7 +99,7 @@ jobs:
working-directory: openmetadata-ui/src/main/resources/ui/
run: yarn --ignore-scripts --frozen-lockfile
- name: Install Playwright Browsers
run: npx playwright@1.44.1 install --with-deps
run: npx playwright@1.48.2 install --with-deps
- name: Run Playwright tests
working-directory: openmetadata-ui/src/main/resources/ui/
run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/py-cli-e2e-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -164,21 +164,21 @@ jobs:
- name: Upload coverage artifact for Python tests
if: matrix.e2e-test == 'python' && steps.python-e2e-test.outcome == 'success' && env.DEBUG == 'false'
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: coverage-${{ matrix.e2e-test }}
path: .coverage

- name: Upload coverage artifact for CLI E2E tests
if: matrix.e2e-test != 'python' && steps.e2e-test.outcome == 'success' && env.DEBUG == 'false'
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: coverage-${{ matrix.e2e-test }}
path: .coverage.${{ matrix.e2e-test }}

- name: Upload tests artifact
if: steps.e2e-test.outcome == 'success' || steps.python-e2e-test.outcome == 'success' && env.DEBUG == 'false'
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: tests-${{ matrix.e2e-test }}
path: ingestion/junit/test-results-*.xml
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/security-scan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ jobs:

- name: Upload Snyk Report HTML files
if: steps.security-report.outcome == 'success'
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: security-report
path: security-report
Expand Down
44 changes: 43 additions & 1 deletion bootstrap/sql/migrations/native/1.6.2/mysql/schemaChanges.sql
Original file line number Diff line number Diff line change
@@ -1,2 +1,44 @@
-- add timestamp index for test case result reindex performance
ALTER TABLE data_quality_data_time_series ADD INDEX `idx_timestamp_desc` (timestamp DESC);
ALTER TABLE data_quality_data_time_series ADD INDEX `idx_timestamp_desc` (timestamp DESC);

CREATE TABLE background_jobs (
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
jobType VARCHAR(256) NOT NULL,
methodName VARCHAR(256) NOT NULL,
jobArgs JSON NOT NULL,
status VARCHAR(50) NOT NULL DEFAULT 'PENDING',
createdBy VARCHAR(256) NOT NULL,
createdAt BIGINT UNSIGNED NOT NULL DEFAULT (UNIX_TIMESTAMP(NOW(3)) * 1000),
updatedAt BIGINT UNSIGNED NOT NULL DEFAULT (UNIX_TIMESTAMP(NOW(3)) * 1000)
);

CREATE INDEX idx_status_createdAt ON background_jobs (status, createdAt);
CREATE INDEX idx_createdBy ON background_jobs (createdBy);
CREATE INDEX idx_status ON background_jobs (status);
CREATE INDEX idx_jobType ON background_jobs (jobType);
CREATE INDEX idx_updatedAt ON background_jobs (updatedAt);

-- rename executable -> basic for test suites
UPDATE test_suite
SET json = JSON_INSERT(
JSON_REMOVE(json, '$.executable'),
'$.basic',
JSON_EXTRACT(json, '$.executable')
)
WHERE JSON_EXTRACT(json, '$.executable') IS NOT NULL;

-- rename executableEntityReference -> basicEntityReference for test suites
UPDATE test_suite
SET json = JSON_INSERT(
JSON_REMOVE(json, '$.executableEntityReference'),
'$.basicEntityReference',
JSON_EXTRACT(json, '$.executableEntityReference')
)
WHERE JSON_EXTRACT(json, '$.executableEntityReference') IS NOT NULL;

-- clean up the testSuites
UPDATE test_case SET json = json_remove(json, '$.testSuites');

-- clean up the testSuites in the version history too
UPDATE entity_extension SET json = json_remove(json, '$.testSuites') WHERE jsonSchema = 'testCase';

44 changes: 44 additions & 0 deletions bootstrap/sql/migrations/native/1.6.2/postgres/schemaChanges.sql
Original file line number Diff line number Diff line change
@@ -1,2 +1,46 @@
-- add timestamp index for test case result reindex performance
CREATE INDEX idx_timestamp_desc ON data_quality_data_time_series (timestamp DESC);

CREATE TABLE background_jobs (
id BIGSERIAL PRIMARY KEY,
jobType VARCHAR(256) NOT NULL,
methodName VARCHAR(256) NOT NULL,
jobArgs JSONB NOT NULL,
status VARCHAR(50) NOT NULL DEFAULT 'PENDING',
createdBy VARCHAR(256) NOT NULL,
createdAt BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT,
updatedAt BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT
);

CREATE INDEX idx_status_createdAt ON background_jobs (status, createdAt);
CREATE INDEX idx_createdBy ON background_jobs (createdBy);
CREATE INDEX idx_status ON background_jobs (status);
CREATE INDEX idx_jobType ON background_jobs (jobType);
CREATE INDEX idx_updatedAt ON background_jobs (updatedAt);

-- rename executable -> basic for test suites
UPDATE test_suite
SET json = jsonb_set(
json::jsonb #- '{executable}',
'{basic}',
(json #> '{executable}')::jsonb,
true
)
WHERE json #>> '{executable}' IS NOT NULL;

-- rename executableEntityReference -> basicEntityReference for test suites
UPDATE test_suite
SET json = jsonb_set(
json::jsonb #- '{executableEntityReference}',
'{basicEntityReference}',
(json #> '{executableEntityReference}')::jsonb,
true
)
WHERE json #>> '{executableEntityReference}' IS NOT NULL;

-- clean up the testSuites
UPDATE test_case SET json = json::jsonb #- '{testSuites}';

-- clean up the testSuites in the version history too
UPDATE entity_extension SET json = json::jsonb #- '{testSuites}' WHERE jsonSchema = 'testCase';

Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
UPDATE workflow_definition_entity
SET json = JSON_SET(json, '$.trigger.type', 'eventBasedEntity')
WHERE JSON_EXTRACT(json, '$.trigger.type') = 'eventBasedEntityWorkflow';

UPDATE workflow_definition_entity
SET json = JSON_SET(json, '$.trigger.type', 'periodicBatchEntity')
WHERE JSON_EXTRACT(json, '$.trigger.type') = 'periodicBatchEntityWorkflow';
3 changes: 3 additions & 0 deletions bootstrap/sql/migrations/native/1.7.0/mysql/schemaChanges.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
UPDATE workflow_definition_entity
SET json = JSON_REMOVE(json, '$.type')
WHERE JSON_EXTRACT(json, '$.type') IS NOT NULL;
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
UPDATE workflow_definition_entity
SET json = jsonb_set(json, '{trigger,type}', '"eventBasedEntity"')
WHERE json->'trigger'->>'type' = 'eventBasedEntityWorkflow';

UPDATE workflow_definition_entity
SET json = jsonb_set(json, '{trigger,type}', '"periodicBatchEntity"')
WHERE json->'trigger'->>'type' = 'periodicBatchEntityWorkflow';
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
UPDATE workflow_definition_entity
SET json = json - 'type'
WHERE json->>'type' IS NOT NULL;
Loading

0 comments on commit 1818a1f

Please sign in to comment.