Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
2 changes: 1 addition & 1 deletion .github/ISSUE_TEMPLATE/config.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0

blank_issues_enabled: false
Expand Down
2 changes: 1 addition & 1 deletion .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0

# To get started with Dependabot version updates, you'll need to specify which
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/autoupdate.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0

name: autoupdate
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/codacy.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0

# This workflow uses actions that are not certified by GitHub.
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/dependabot.autoapprove.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0

name: Auto approve
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/dependabot.automerge.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0

name: automerge
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/format.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0

name: Google Java Format
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pr-title-checker.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0

name: "PR Title Checker"
Expand Down
2 changes: 1 addition & 1 deletion .gitlab-ci.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0

# default image used by all pipelines if not overridden per job
Expand Down
16 changes: 8 additions & 8 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0

repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
rev: v6.0.0
hooks:
- id: check-yaml
args: [ '--unsafe' ]
Expand All @@ -21,8 +21,8 @@ repos:
- id: requirements-txt-fixer
- id: detect-private-key
- id: fix-byte-order-marker
- repo: https://github.com/psf/black
rev: 23.11.0
- repo: https://github.com/psf/black-pre-commit-mirror
rev: 26.1.0
hooks:
- id: black
language_version: python3
Expand All @@ -46,13 +46,13 @@ repos:
# - id: mypy
# files: ^(src/|tests/|plugins/)
- repo: https://github.com/asottile/reorder-python-imports
rev: v3.12.0
rev: v3.16.0
hooks:
- id: reorder-python-imports
args: [ --py37-plus, '--application-directories=.:src' ]
# use latest python syntax
- repo: https://github.com/asottile/pyupgrade
rev: v3.15.0
rev: v3.21.2
hooks:
- id: pyupgrade
args: [ --py37-plus ]
Expand All @@ -65,7 +65,7 @@ repos:
# args: [ -s, B101 ]
# add copyright notice
- repo: https://github.com/Lucas-C/pre-commit-hooks
rev: v1.5.4
rev: v1.5.6
hooks:
- id: insert-license
files: \.java$
Expand Down Expand Up @@ -144,7 +144,7 @@ repos:
- --license-filepath
- NOTICE.txt
- repo: https://github.com/pre-commit/mirrors-prettier
rev: v3.1.0
rev: v4.0.0-alpha.8
hooks:
- id: prettier
types_or: [scss, css, javascript, ts, html]
2 changes: 1 addition & 1 deletion cicd/.gitlab-ci-lib.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0

# This is a shared library with common util gitlab "jobs" that can be reused across any project
Expand Down
2 changes: 1 addition & 1 deletion cicd/build.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/bin/bash -e
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
Expand Down
2 changes: 1 addition & 1 deletion cicd/docker_parse.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0

# Parses a Docker image name and extracts information such as the registry, username, repository, and tag.
Expand Down
2 changes: 1 addition & 1 deletion cicd/docker_push_vdk.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env bash

# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0

# This Bash script provides a utility to push a containers image to multiple container registries.
Expand Down
2 changes: 1 addition & 1 deletion cicd/notify.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/bin/bash -e
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0

curl -X POST -H "Content-type: application/json" --data "{\"text\":\"$1\"}" $2
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
from vdk.api.job_input import IJobInput

Expand Down
12 changes: 4 additions & 8 deletions examples/airflow-example/airflow-trino-job1/10_insert_data.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
import json
import pathlib
Expand All @@ -17,12 +17,9 @@ def run(job_input: IJobInput):
rows = [tuple(i.values()) for i in data]
insert_query = """
INSERT INTO memory.default.test_airflow_one VALUES
""" + ", ".join(
str(i) for i in rows
)
""" + ", ".join(str(i) for i in rows)

job_input.execute_query(
"""
job_input.execute_query("""
CREATE TABLE IF NOT EXISTS memory.default.test_airflow_one
(
id varchar,
Expand All @@ -32,8 +29,7 @@ def run(job_input: IJobInput):
country varchar,
phone varchar
)
"""
)
""")

job_input.execute_query(insert_query)

Expand Down
12 changes: 4 additions & 8 deletions examples/airflow-example/airflow-trino-job2/10_insert_data.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
import json
import pathlib
Expand All @@ -17,12 +17,9 @@ def run(job_input: IJobInput):
rows = [tuple(i.values()) for i in data]
insert_query = """
INSERT INTO memory.default.test_airflow_two VALUES
""" + ", ".join(
str(i) for i in rows
)
""" + ", ".join(str(i) for i in rows)

job_input.execute_query(
"""
job_input.execute_query("""
CREATE TABLE IF NOT EXISTS memory.default.test_airflow_two
(
id integer,
Expand All @@ -32,8 +29,7 @@ def run(job_input: IJobInput):
country varchar,
phone varchar
)
"""
)
""")

job_input.execute_query(insert_query)

Expand Down
2 changes: 1 addition & 1 deletion examples/airflow-example/dags/airflow_example_dag.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
from datetime import datetime

Expand Down
2 changes: 1 addition & 1 deletion examples/chunker/00_properties.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
# Copyright 2021-2024 VMware, Inc.
# SPDX-License-Identifier: Apache-2.0
Expand Down
2 changes: 1 addition & 1 deletion examples/chunker/10_chunk_data.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
# Copyright 2021-2024 VMware, Inc.
# SPDX-License-Identifier: Apache-2.0
Expand Down
2 changes: 1 addition & 1 deletion examples/chunker/config.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
# Copyright 2021-2024 VMware, Inc.
# SPDX-License-Identifier: Apache-2.0
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
import logging
import os
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
import logging
import os
Expand Down
2 changes: 1 addition & 1 deletion examples/confluence-reader/confluence_document.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0


Expand Down
2 changes: 1 addition & 1 deletion examples/confluence-reader/fetch_confluence_space.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
import logging
import os
Expand Down
3 changes: 1 addition & 2 deletions examples/dag-example/example-dag/example_dag.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
from vdk.plugin.dag.dag_runner import DagInput


JOBS_RUN_ORDER = [
{
"job_name": "ingest-job1",
Expand Down
12 changes: 4 additions & 8 deletions examples/dag-example/ingest-job1/10_insert_data.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
import json
import pathlib
Expand All @@ -17,12 +17,9 @@ def run(job_input: IJobInput):
rows = [tuple(i.values()) for i in data]
insert_query = """
INSERT INTO memory.default.test_dag_one VALUES
""" + ", ".join(
str(i) for i in rows
)
""" + ", ".join(str(i) for i in rows)

job_input.execute_query(
"""
job_input.execute_query("""
CREATE TABLE IF NOT EXISTS memory.default.test_dag_one
(
id varchar,
Expand All @@ -32,8 +29,7 @@ def run(job_input: IJobInput):
country varchar,
phone varchar
)
"""
)
""")

job_input.execute_query(insert_query)

Expand Down
12 changes: 4 additions & 8 deletions examples/dag-example/ingest-job2/10_insert_data.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
import json
import pathlib
Expand All @@ -17,12 +17,9 @@ def run(job_input: IJobInput):
rows = [tuple(i.values()) for i in data]
insert_query = """
INSERT INTO memory.default.test_dag_two VALUES
""" + ", ".join(
str(i) for i in rows
)
""" + ", ".join(str(i) for i in rows)

job_input.execute_query(
"""
job_input.execute_query("""
CREATE TABLE IF NOT EXISTS memory.default.test_dag_two
(
id integer,
Expand All @@ -32,8 +29,7 @@ def run(job_input: IJobInput):
country varchar,
phone varchar
)
"""
)
""")

job_input.execute_query(insert_query)

Expand Down
2 changes: 1 addition & 1 deletion examples/dag-example/read-data-job/10_read.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
from vdk.api.job_input import IJobInput

Expand Down
3 changes: 1 addition & 2 deletions examples/dag-with-args-example/dag-job/dag_job.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
from vdk.plugin.dag.dag_runner import DagInput


JOBS_RUN_ORDER = [
{
"job_name": "ingest-job-table-one",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
import json
import pathlib
Expand All @@ -20,9 +20,7 @@ def run(job_input: IJobInput):
rows = [tuple(i.values()) for i in data]
insert_query = f"""
INSERT INTO {db_schema}.{db_table} VALUES
""" + ", ".join(
str(i) for i in rows
)
""" + ", ".join(str(i) for i in rows)

create_query = f"""
CREATE TABLE IF NOT EXISTS {db_schema}.{db_table}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
import json
import pathlib
Expand All @@ -20,9 +20,7 @@ def run(job_input: IJobInput):
rows = [tuple(i.values()) for i in data]
insert_query = f"""
INSERT INTO {db_schema}.{db_table} VALUES
""" + ", ".join(
str(i) for i in rows
)
""" + ", ".join(str(i) for i in rows)

create_query = f"""
CREATE TABLE IF NOT EXISTS {db_schema}.{db_table}
Expand Down
2 changes: 1 addition & 1 deletion examples/dag-with-args-example/read-job-canada/10_read.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
from vdk.api.job_input import IJobInput

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023-2025 Broadcom
# Copyright 2023-2026 Broadcom
# SPDX-License-Identifier: Apache-2.0
from vdk.api.job_input import IJobInput

Expand Down
Loading
Loading