From b852939d82a776aada61aae445c3df718007c105 Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Fri, 21 Feb 2025 16:54:07 -0500 Subject: [PATCH 1/7] Add pyproject.toml to replace most of setup.py setup.py is still needed for the cython build until tool.setuptools.ext.ext-modules is no longer experimental. --- .github/workflows/tests.yml | 1 - LIFECYCLE.md | 3 ++- __init__.py | 0 jsonobject/__init__.py | 1 + pyproject.toml | 34 ++++++++++++++++++++++++++++++++++ scripts/install_cython.sh | 7 ++++--- setup.cfg | 2 -- setup.py | 30 ++---------------------------- 8 files changed, 43 insertions(+), 35 deletions(-) delete mode 100644 __init__.py create mode 100644 pyproject.toml delete mode 100644 setup.cfg diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1fd1c82..dbeb645 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,7 +26,6 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install setuptools python -m pip install -e . - name: Run tests run: | diff --git a/LIFECYCLE.md b/LIFECYCLE.md index 0024119..a3487db 100644 --- a/LIFECYCLE.md +++ b/LIFECYCLE.md @@ -38,7 +38,8 @@ This section contains instructions for the Dimagi team member performing the rel ## Bump version & update CHANGES.md -In a single PR, bump the version number and update CHANGES.md to include release notes for this new version. +In a single PR, bump the version number in `jsonobject/__init__.py` and update +CHANGES.md to include release notes for this new version. ### Pick a version number diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/jsonobject/__init__.py b/jsonobject/__init__.py index cb479bc..9a41eed 100644 --- a/jsonobject/__init__.py +++ b/jsonobject/__init__.py @@ -2,6 +2,7 @@ from .properties import * from .api import JsonObject +__version__ = '2.2.0' __all__ = [ 'IntegerProperty', 'FloatProperty', 'DecimalProperty', 'StringProperty', 'BooleanProperty', diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6f13c1b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "jsonobject" +description = "A library for dealing with JSON as python objects" +authors = [{name = "Danny Roberts", email = "droberts@dimagi.com"}] +license = {file = "LICENSE"} +readme = {file = "README.md", content-type = "text/markdown"} +dynamic = ["version"] +requires-python = ">= 3.9" +classifiers = [ + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "License :: OSI Approved :: BSD License", +] + +[project.urls] +Home = "https://github.com/dimagi/jsonobject" + +[build-system] +requires = [ + "setuptools>=75", + "Cython>=3.0.0,<4.0.0", +] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = ["jsonobject"] + +[tool.setuptools.dynamic] +version = {attr = "jsonobject.__version__"} diff --git a/scripts/install_cython.sh b/scripts/install_cython.sh index a00a59b..98afef4 100755 --- a/scripts/install_cython.sh +++ b/scripts/install_cython.sh @@ -1,6 +1,7 @@ #! /bin/bash -# grep setup.py for the pinned version of cython -PINNED_CYTHON=$(grep -oE 'cython>=[0-9]+\.[0-9]+\.[0-9]+,<[0-9]+\.[0-9]+\.[0-9]+' setup.py) +# grep pyproject.toml for the pinned version of cython +PINNED_CYTHON=$(grep -oE 'Cython>?=[^"]+' pyproject.toml) -pip install $PINNED_CYTHON +echo "Installing $PINNED_CYTHON" +pip install $PINNED_CYTHON setuptools diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 0c9e0fc..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -license_file = LICENSE diff --git a/setup.py b/setup.py index 8c77850..69198d6 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ from setuptools import setup -import io from setuptools.extension import Extension @@ -20,40 +19,15 @@ Extension('jsonobject.utils', ["jsonobject/utils" + ext],), ] -CYTHON_REQUIRES = ['cython>=3.0.0,<4.0.0'] if USE_CYTHON: from Cython.Build import cythonize extensions = cythonize(extensions, compiler_directives={"language_level" : "3str"}) else: print("You are running without Cython installed. It is highly recommended to run\n" - " pip install {}\n" - "before you continue".format(' '.join(CYTHON_REQUIRES))) - - -with io.open('README.md', 'rt', encoding="utf-8") as readme_file: - long_description = readme_file.read() - + " ./scripts/install_cython.sh\n" + "before you continue") setup( name='jsonobject', - version='2.2.0', - author='Danny Roberts', - author_email='droberts@dimagi.com', - description='A library for dealing with JSON as python objects', - long_description=long_description, - long_description_content_type='text/markdown', - url='https://github.com/dimagi/jsonobject', - packages=['jsonobject'], - setup_requires=CYTHON_REQUIRES, ext_modules=extensions, - classifiers=[ - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'License :: OSI Approved :: BSD License', - ], ) From 0dd20545de60b8460cb022b500f968a63b2aca2a Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Fri, 21 Feb 2025 16:46:25 -0500 Subject: [PATCH 2/7] Automate python version matrix on gitub actions pyproject.toml is the source of truth for supported Python versions. No other files need to be kept in sync. --- .github/workflows/tests.yml | 21 +++++++++++++++++++-- pyproject.toml | 4 ++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dbeb645..0754103 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,13 +9,30 @@ on: pull_request: jobs: - build: + configure: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Read Python versions from pyproject.toml + id: read-versions + # produces output like: python_versions=[ "3.9", "3.10", "3.11", "3.12" ] + run: >- + echo "python_versions=$( + grep -oP '(?<=Language :: Python :: )\d\.\d+' pyproject.toml + | jq --raw-input . + | jq --slurp . + | tr '\n' ' ' + )" >> $GITHUB_OUTPUT + outputs: + python_versions: ${{ steps.read-versions.outputs.python_versions }} + build: + needs: [configure] runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ${{ fromJSON(needs.configure.outputs.python_versions) }} steps: - uses: actions/checkout@v2 diff --git a/pyproject.toml b/pyproject.toml index 6f13c1b..97c4f33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,11 +9,15 @@ requires-python = ">= 3.9" classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3", + + # The following classifiers are parsed by Github Actions workflows. + # Precise formatting is important (no extra spaces, etc.) "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "License :: OSI Approved :: BSD License", ] From 569a94ff1468adb90c646471990764cd47b1724f Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Fri, 21 Feb 2025 16:47:12 -0500 Subject: [PATCH 3/7] Update github action versions --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0754103..70459cd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,9 +35,9 @@ jobs: python-version: ${{ fromJSON(needs.configure.outputs.python_versions) }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies From 65927e3ab569670a4e032794e15ff7c2e28549fb Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Fri, 21 Feb 2025 11:48:59 -0500 Subject: [PATCH 4/7] Automatic publish to PyPI on push master or tag --- .github/workflows/pypi.yml | 135 +++++++++++++++++++++++++++++++++++++ LIFECYCLE.md | 9 ++- version.py | 64 ++++++++++++++++++ 3 files changed, 205 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/pypi.yml create mode 100644 version.py diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml new file mode 100644 index 0000000..53c991b --- /dev/null +++ b/.github/workflows/pypi.yml @@ -0,0 +1,135 @@ +name: Build wheels and publish to PyPI + +on: + push: + branches: + - master + tags: + - "v*" + workflow_dispatch: + +jobs: + configure: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Read Python versions from pyproject.toml + id: read-versions + # produces output like: python_versions=39,310,311,312,313 + run: >- + echo "python_versions=$( + grep -oP '(?<=Language :: Python :: )\d.\d+' pyproject.toml + | sed 's/\.//' + | tr '\n' ',' + | sed 's/,$//' + )" >> $GITHUB_OUTPUT + outputs: + python_versions: ${{ steps.read-versions.outputs.python_versions }} + + build_sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check version + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + run: python version.py check "${{ github.ref }}" + - name: Add untagged version suffix + if: ${{ ! startsWith(github.ref, 'refs/tags/v') }} + run: python version.py update + - name: Build sdist + run: pipx run build --sdist + - uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist + + choose_linux_wheel_types: + name: Decide which wheel types to build + runs-on: ubuntu-latest + steps: + - id: manylinux_x86_64 + run: echo "wheel_types=manylinux_x86_64" >> $GITHUB_OUTPUT + - id: musllinux_x86_64 + run: echo "wheel_types=musllinux_x86_64" >> $GITHUB_OUTPUT + outputs: + wheel_types: ${{ toJSON(steps.*.outputs.wheel_types) }} + + build_linux_wheels: + needs: [configure, choose_linux_wheel_types, build_sdist] + name: ${{ matrix.wheel_type }} wheels + runs-on: ubuntu-latest + strategy: + matrix: + wheel_type: ${{ fromJSON(needs.choose_linux_wheel_types.outputs.wheel_types) }} + steps: + - uses: actions/download-artifact@v4 + with: + name: sdist + path: dist + - name: Extract sdist + run: | + tar zxvf dist/*.tar.gz --strip-components=1 + - uses: docker/setup-qemu-action@v3 + if: runner.os == 'Linux' + name: Set up QEMU + - name: Build wheels + uses: pypa/cibuildwheel@v2.22.0 + env: + CIBW_BUILD: cp{${{ needs.configure.outputs.python_versions }}}-${{ matrix.wheel_type }} + CIBW_ARCHS_LINUX: auto + CIBW_BEFORE_BUILD: cd {project}; pip install -e . # is there a better way to build the .so files? + CIBW_TEST_COMMAND: cd {project}; python -m unittest + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.wheel_type }}-wheels + path: ./wheelhouse/*.whl + + pypi-publish: + name: Upload release to PyPI + needs: [build_sdist, build_linux_wheels] + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') + environment: + name: pypi + url: https://pypi.org/p/jsonobject + permissions: + id-token: write + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + # with no name set, it downloads all of the artifacts + path: dist/ + - run: | + mv dist/sdist/*.tar.gz dist/ + mv dist/*-wheels/*.whl dist/ + rmdir dist/{sdist,*-wheels} + ls -R dist + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + pypi-test-publish: + name: Upload release to test PyPI + needs: [build_sdist, build_linux_wheels] + runs-on: ubuntu-latest + environment: + name: testpypi + url: https://test.pypi.org/p/jsonobject + permissions: + id-token: write + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + # with no name set, it downloads all of the artifacts + path: dist/ + - run: | + mv dist/sdist/*.tar.gz dist/ + mv dist/*-wheels/*.whl dist/ + rmdir dist/{sdist,*-wheels} + ls -R dist + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ diff --git a/LIFECYCLE.md b/LIFECYCLE.md index a3487db..4f2d4e4 100644 --- a/LIFECYCLE.md +++ b/LIFECYCLE.md @@ -63,6 +63,9 @@ Once this PR is reviewed and merged, move on to the steps to release the update ## Release the new version -To push the package to pypi, we follow Dimagi's internal documentation. -Follow the steps in https://confluence.dimagi.com/display/saas/Python+Packaging+Crash+Course -to release. +To push the package to pypi, create a git tag named "vX.Y.Z" using the version +in `jsonobject/__init__.py` and push it to Github. + +A test release is pushed to test.pypi.com/projects/jsonobject on each push/merge +to master. A test release may also be published on-demand for any branch with +[workflow dispatch](https://github.com/dimagi/jsonobject/actions/workflows/pypi.yml). diff --git a/version.py b/version.py new file mode 100644 index 0000000..617da3f --- /dev/null +++ b/version.py @@ -0,0 +1,64 @@ +"""Check or update version in __init__.py + +Usage: + python version.py check GITHUB_REF + python version.py update [GITHUB_SHA] +""" +import re +import sys +import importlib +from datetime import UTC, datetime +from pathlib import Path + +PACKAGE_NAME = "jsonobject" + + +def main(argv=sys.argv): + if len(argv) < 2: + sys.exit(f"usage: {argv[0]} (check|update) [...]") + cmd, *args = sys.argv[1:] + if cmd in COMMANDS: + COMMANDS[cmd](*args) + else: + sys.exit(f"unknown arguments: {argv[1:]}") + + +def check(ref): + pkg = importlib.import_module(PACKAGE_NAME) + if not ref.startswith("refs/tags/v"): + sys.exit(f"unexpected ref: {ref}") + version = ref.removeprefix("refs/tags/v") + if version != pkg.__version__: + sys.exit(f"version mismatch: {version} != {pkg.__version__}") + + +def update(sha=""): + """Add a timestamped dev version qualifier to the current version + + Note: PyPI does not allow the "local" version component, which + is where this puts the git sha. Do not pass a sha argument when + updating the version for a PyPI release. + PyPI error: The use of local versions ... is not allowed + """ + path = Path(__file__).parent / PACKAGE_NAME / "__init__.py" + vexpr = re.compile(r"""(?<=^__version__ = )['"](.+)['"]$""", flags=re.M) + with open(path, "r+") as file: + text = file.read() + match = vexpr.search(text) + if not match: + sys.exit(f"{PACKAGE_NAME}.__version__ not found") + devv = datetime.now(UTC).strftime("%Y%m%d%H%M%S") + if sha: + devv += f"+{sha[:7]}" + version = f"{match.group(1)}.dev{devv}" + print("new version:", version) + file.seek(0) + file.write(vexpr.sub(repr(version), text)) + file.truncate() + + +COMMANDS = {"check": check, "update": update} + + +if __name__ == "__main__": + main() From d498e565b35649038e736db262e5127391b24f91 Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Mon, 24 Feb 2025 11:43:45 -0500 Subject: [PATCH 5/7] Publish dev releases to pypi.org This is a workaround for the fact that https://test.pypi.org/p/jsonobject has been claimed by another project. Dev releases will not be downloaded by pip unless the `--pre` argument is used, and the latest stable release will be displayed on https://pypi.org/p/jsonobject by default. --- .github/workflows/pypi.yml | 51 +++++++++++++++++++------------------- LIFECYCLE.md | 4 +-- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 53c991b..4eacbac 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -89,7 +89,7 @@ jobs: name: Upload release to PyPI needs: [build_sdist, build_linux_wheels] runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/v') + #if: startsWith(github.ref, 'refs/tags/v') # removed until pypi-test-publish is working environment: name: pypi url: https://pypi.org/p/jsonobject @@ -109,27 +109,28 @@ jobs: - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - pypi-test-publish: - name: Upload release to test PyPI - needs: [build_sdist, build_linux_wheels] - runs-on: ubuntu-latest - environment: - name: testpypi - url: https://test.pypi.org/p/jsonobject - permissions: - id-token: write - steps: - - name: Download all the dists - uses: actions/download-artifact@v4 - with: - # with no name set, it downloads all of the artifacts - path: dist/ - - run: | - mv dist/sdist/*.tar.gz dist/ - mv dist/*-wheels/*.whl dist/ - rmdir dist/{sdist,*-wheels} - ls -R dist - - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: https://test.pypi.org/legacy/ + # https://github.com/finpassbr/json-object/issues/1 + #pypi-test-publish: + # name: Upload release to test PyPI + # needs: [build_sdist, build_linux_wheels] + # runs-on: ubuntu-latest + # environment: + # name: testpypi + # url: https://test.pypi.org/p/jsonobject + # permissions: + # id-token: write + # steps: + # - name: Download all the dists + # uses: actions/download-artifact@v4 + # with: + # # with no name set, it downloads all of the artifacts + # path: dist/ + # - run: | + # mv dist/sdist/*.tar.gz dist/ + # mv dist/*-wheels/*.whl dist/ + # rmdir dist/{sdist,*-wheels} + # ls -R dist + # - name: Publish package distributions to PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # repository-url: https://test.pypi.org/legacy/ diff --git a/LIFECYCLE.md b/LIFECYCLE.md index 4f2d4e4..656759b 100644 --- a/LIFECYCLE.md +++ b/LIFECYCLE.md @@ -66,6 +66,6 @@ Once this PR is reviewed and merged, move on to the steps to release the update To push the package to pypi, create a git tag named "vX.Y.Z" using the version in `jsonobject/__init__.py` and push it to Github. -A test release is pushed to test.pypi.com/projects/jsonobject on each push/merge -to master. A test release may also be published on-demand for any branch with +A dev release is pushed to pypi.com/p/jsonobject/#history on each push/merge to +master. A dev release may also be published on-demand for any branch with [workflow dispatch](https://github.com/dimagi/jsonobject/actions/workflows/pypi.yml). From 769fa4a429d33a8d5844d40f189c831f265fc537 Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Mon, 24 Feb 2025 12:04:37 -0500 Subject: [PATCH 6/7] Bump version to 2.3.0 and update change log --- CHANGES.md | 14 ++++++++++++++ jsonobject/__init__.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index aa0d98c..d7f5287 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,20 @@ No significant changes since the last release +## 2.3.0 + +| Released on | Released by | +|-------------|---------------| +| UNRELEASED | @millerdev | + +- Improve build and automate push to PyPI (https://github.com/dimagi/jsonobject/pull/236) + - Add pyproject.toml to replace most of setup.py + - Automate python version matrix on gitub actions + - Update github action versions + - Publish releases to pypi.org +- Build C files with Cython 3.0.12 (https://github.com/dimagi/jsonobject/pull/235) + - Add support for Python 3.13 + ## 2.2.0 | Released on | Released by | diff --git a/jsonobject/__init__.py b/jsonobject/__init__.py index 9a41eed..0349521 100644 --- a/jsonobject/__init__.py +++ b/jsonobject/__init__.py @@ -2,7 +2,7 @@ from .properties import * from .api import JsonObject -__version__ = '2.2.0' +__version__ = '2.3.0' __all__ = [ 'IntegerProperty', 'FloatProperty', 'DecimalProperty', 'StringProperty', 'BooleanProperty', From 1c0f0526d4092a99a572d224ecc40dc86de94184 Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Mon, 24 Feb 2025 15:12:19 -0500 Subject: [PATCH 7/7] Set 2.3.0 release date --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d7f5287..7295df3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,7 +9,7 @@ No significant changes since the last release | Released on | Released by | |-------------|---------------| -| UNRELEASED | @millerdev | +| 2025-02-24 | @millerdev | - Improve build and automate push to PyPI (https://github.com/dimagi/jsonobject/pull/236) - Add pyproject.toml to replace most of setup.py