Skip to content

Commit 206c2cd

Browse files
SigureMoSigureMo
committed
[Compat] Compatible with PaddlePaddle
wrap convert list to cpu tensor Add workflow for paddlepaddle wheel build (#1) add 3.13 workflow set ci target branch to paddle fix artifact name update name use auditwheel exclude ffmpeg libs exclude paddle libs exclude all
1 parent d63504c commit 206c2cd

File tree

14 files changed

+534
-42
lines changed

14 files changed

+534
-42
lines changed

‎.editorconfig‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# EditorConfig
2+
# https://editorconfig.org/
3+
4+
root = true
5+
6+
[*]
7+
indent_style = space
8+
indent_size = 2
9+
end_of_line = lf
10+
charset = utf-8
11+
trim_trailing_whitespace = true
12+
insert_final_newline = true
13+
14+
[*.{py,sh,ipynb}]
15+
indent_size = 4
16+
17+
[*{.md,rst}]
18+
indent_size = 3

‎.github/workflows/paddle_wheel.yaml‎

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
name: Build Wheels for Paddle
2+
3+
on:
4+
push:
5+
branches: [paddle]
6+
tags: ["v*"]
7+
pull_request:
8+
merge_group:
9+
workflow_dispatch:
10+
11+
concurrency:
12+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
13+
cancel-in-progress: true
14+
15+
permissions:
16+
id-token: write
17+
contents: write
18+
19+
defaults:
20+
run:
21+
shell: bash -l -eo pipefail {0}
22+
23+
jobs:
24+
build-paddlecodec-wheel:
25+
runs-on: ubuntu-latest
26+
container:
27+
image: pytorch/manylinux2_28-builder:cpu
28+
strategy:
29+
fail-fast: false
30+
matrix:
31+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
32+
permissions:
33+
id-token: write
34+
contents: read
35+
steps:
36+
- name: Checkout repository
37+
uses: actions/checkout@v6
38+
39+
- name: Setup conda environment
40+
uses: conda-incubator/setup-miniconda@v3
41+
with:
42+
auto-update-conda: true
43+
miniforge-version: latest
44+
activate-environment: build
45+
python-version: ${{ matrix.python-version }}
46+
47+
- name: Install build dependencies
48+
run: |
49+
python -m pip install --upgrade pip
50+
pip install build wheel setuptools
51+
52+
- name: Install PaddlePaddle nightly
53+
run: |
54+
pip install --pre paddlepaddle -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/
55+
56+
- name: Run pre-build script
57+
run: |
58+
bash packaging/pre_build_script.sh
59+
60+
- name: Build wheel
61+
run: |
62+
# Use pre-built FFmpeg from PyTorch S3
63+
export BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1
64+
export TORCHCODEC_CMAKE_BUILD_DIR=$(pwd)/build_cmake
65+
python -m build --wheel -vvv --no-isolation
66+
67+
- name: Repair wheel
68+
run: |
69+
pip install auditwheel
70+
71+
# 1. Extract internal libraries from the wheel to a temporary directory
72+
# This allows auditwheel to find them when checking dependencies
73+
mkdir -p temp_libs
74+
unzip -j dist/*.whl "torchcodec/*.so" -d temp_libs || true
75+
76+
# 2. Prepare LD_LIBRARY_PATH
77+
# FFmpeg libraries
78+
FFMPEG_LIB_PATHS=$(find $(pwd)/build_cmake/_deps -type d -name "lib" | tr '\n' ':')
79+
# PaddlePaddle libraries
80+
PADDLE_PATH=$(python -c "import paddle; print(paddle.__path__[0])")
81+
PADDLE_LIB_PATHS="$PADDLE_PATH/base:$PADDLE_PATH/libs"
82+
# Wheel internal libraries
83+
INTERNAL_LIB_PATH=$(pwd)/temp_libs
84+
85+
export LD_LIBRARY_PATH=${FFMPEG_LIB_PATHS}${PADDLE_LIB_PATHS}:${INTERNAL_LIB_PATH}:${LD_LIBRARY_PATH}
86+
87+
# 3. Repair wheel with auditwheel
88+
# We exclude all external libraries because we want to rely on system libraries (like FFmpeg)
89+
# or libraries provided by other packages (like PaddlePaddle).
90+
# auditwheel 6.1.0+ supports wildcards in --exclude.
91+
auditwheel repair dist/*.whl --plat manylinux_2_28_x86_64 -w wheelhouse/ --exclude "*"
92+
93+
# Cleanup
94+
rm -rf temp_libs
95+
rm dist/*.whl
96+
mv wheelhouse/*.whl dist/
97+
rmdir wheelhouse
98+
99+
- name: Upload wheel artifact
100+
uses: actions/upload-artifact@v5
101+
with:
102+
name: paddlecodec-wheel-linux-py${{ matrix.python-version }}
103+
path: dist/*.whl
104+
105+
- name: Run post-build script
106+
run: |
107+
bash packaging/post_build_script.sh
108+
109+
- name: List wheel contents
110+
run: |
111+
wheel_path=$(find dist -type f -name "*.whl")
112+
echo "Wheel path: $wheel_path"
113+
unzip -l $wheel_path
114+
115+
test-paddlecodec-wheel:
116+
needs: build-paddlecodec-wheel
117+
runs-on: ubuntu-latest
118+
strategy:
119+
fail-fast: false
120+
matrix:
121+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
122+
# FFmpeg 8.0 depends on libopenvino.so.2520, PaddlePaddle CPU depends on libopenvino.so.2500
123+
# There has some conflict causing test failures, but it works with PaddlePaddle GPU.
124+
# We skip FFmpeg 8.0 tests for PaddlePaddle CPU builds for now.
125+
ffmpeg-version: ["4.4.2", "5.1.2", "6.1.1", "7.0.1"]
126+
steps:
127+
- name: Checkout repository
128+
uses: actions/checkout@v6
129+
130+
- name: Set up Python ${{ matrix.python-version }}
131+
uses: actions/setup-python@v5
132+
with:
133+
python-version: ${{ matrix.python-version }}
134+
135+
- name: Download wheel artifact
136+
uses: actions/download-artifact@v4
137+
with:
138+
name: paddlecodec-wheel-linux-py${{ matrix.python-version }}
139+
path: dist/
140+
141+
- name: Install FFmpeg via conda
142+
uses: conda-incubator/setup-miniconda@v3
143+
with:
144+
auto-update-conda: true
145+
miniforge-version: latest
146+
activate-environment: test
147+
python-version: ${{ matrix.python-version }}
148+
149+
- name: Install FFmpeg from conda-forge
150+
run: |
151+
conda install "ffmpeg=${{ matrix.ffmpeg-version }}" -c conda-forge -y
152+
ffmpeg -version
153+
154+
- name: Install PaddlePaddle nightly in conda env
155+
run: |
156+
pip install --pre paddlepaddle -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/
157+
158+
- name: Install paddlecodec from wheel
159+
run: |
160+
wheel_path=$(find dist -type f -name "*.whl")
161+
echo "Installing $wheel_path"
162+
pip install $wheel_path -vvv
163+
164+
- name: Install test dependencies
165+
run: |
166+
pip install numpy pytest pillow
167+
168+
- name: Delete src folder
169+
run: |
170+
# Delete src/ to ensure we're testing the installed wheel, not source code
171+
rm -rf src/
172+
ls -la
173+
174+
- name: Run tests
175+
run: |
176+
pytest --override-ini="addopts=-v" -s test_paddle
177+
178+
publish-pypi:
179+
runs-on: ubuntu-latest
180+
name: Publish to PyPI
181+
if: "startsWith(github.ref, 'refs/tags/')"
182+
needs:
183+
- test-paddlecodec-wheel
184+
permissions:
185+
id-token: write
186+
187+
steps:
188+
- name: Retrieve release distributions
189+
uses: actions/download-artifact@v6
190+
with:
191+
pattern: paddlecodec-wheel-linux-*
192+
path: dist/
193+
merge-multiple: true
194+
195+
- name: Publish release distributions to PyPI
196+
uses: pypa/gh-action-pypi-publish@release/v1
197+
198+
publish-release:
199+
runs-on: ubuntu-latest
200+
name: Publish to GitHub
201+
if: "startsWith(github.ref, 'refs/tags/')"
202+
needs:
203+
- test-paddlecodec-wheel
204+
permissions:
205+
contents: write
206+
steps:
207+
- uses: actions/download-artifact@v6
208+
with:
209+
pattern: paddlecodec-wheel-linux-*
210+
path: dist/
211+
merge-multiple: true
212+
- name: Get tag name
213+
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
214+
- name: Publish to GitHub
215+
uses: softprops/action-gh-release@v2
216+
with:
217+
draft: true
218+
files: dist/*
219+
tag_name: ${{ env.RELEASE_VERSION }}

‎examples/decoding/sampling.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919
# plotting utility. You can ignore that part and jump right below to
2020
# :ref:`sampling_tuto_start`.
2121

22+
import paddle
23+
paddle.compat.enable_torch_proxy(scope={"torchcodec"})
24+
2225
from typing import Optional
2326
import torch
2427
import requests

‎pyproject.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
[project]
2-
name = "torchcodec"
2+
name = "paddlecodec"
33
description = "A video decoder for PyTorch"
44
readme = "README.md"
55
requires-python = ">=3.8"
66
license-files = ["LICENSE"]
77
authors = [
8-
{ name = "PyTorch Team", email = "packages@pytorch.org" },
8+
{ name = "PaddlePaddle Team", email = "Paddle-better@baidu.com" },
99
]
1010
dynamic = ["version"]
1111

‎setup.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
import sys
4747
from pathlib import Path
4848

49-
import torch
49+
import paddle
5050
from setuptools import Extension, setup
5151
from setuptools.command.build_ext import build_ext
5252

@@ -109,7 +109,7 @@ def build_extension(self, ext):
109109
def _build_all_extensions_with_cmake(self):
110110
# Note that self.debug is True when you invoke setup.py like this:
111111
# python setup.py build_ext --debug install
112-
torch_dir = Path(torch.utils.cmake_prefix_path) / "Torch"
112+
# torch_dir = Path(torch.utils.cmake_prefix_path) / "Torch"
113113
cmake_build_type = os.environ.get("CMAKE_BUILD_TYPE", "Release")
114114
enable_cuda = os.environ.get("ENABLE_CUDA", "")
115115
torchcodec_disable_compile_warning_as_error = os.environ.get(
@@ -118,7 +118,8 @@ def _build_all_extensions_with_cmake(self):
118118
python_version = sys.version_info
119119
cmake_args = [
120120
f"-DCMAKE_INSTALL_PREFIX={self._install_prefix}",
121-
f"-DTorch_DIR={torch_dir}",
121+
# f"-DTorch_DIR={torch_dir}",
122+
f"-DPADDLE_PATH={paddle.__path__[0]}",
122123
"-DCMAKE_VERBOSE_MAKEFILE=ON",
123124
f"-DCMAKE_BUILD_TYPE={cmake_build_type}",
124125
f"-DPYTHON_VERSION={python_version.major}.{python_version.minor}",

‎src/torchcodec/_core/AVIOTensorContext.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,8 +123,10 @@ AVIOToTensorContext::AVIOToTensorContext()
123123
}
124124

125125
torch::Tensor AVIOToTensorContext::getOutputTensor() {
126-
return tensorContext_.data.narrow(
127-
/*dim=*/0, /*start=*/0, /*length=*/tensorContext_.max_pos);
126+
throw std::runtime_error(
127+
"AVIOToTensorContext::getOutputTensor is not implemented yet.");
128+
// return tensorContext_.data.narrow(
129+
// /*dim=*/0, /*start=*/0, /*length=*/tensorContext_.max_pos);
128130
}
129131

130132
} // namespace facebook::torchcodec

‎src/torchcodec/_core/CMakeLists.txt‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,29 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
55

66
set(PYBIND11_FINDPYTHON ON)
77
find_package(pybind11 REQUIRED)
8-
find_package(Torch REQUIRED)
8+
#find_package(Torch REQUIRED)
9+
set(TORCH_CXX_FLAGS "")
10+
11+
set(
12+
TORCH_LIBRARIES
13+
"${PADDLE_PATH}/base/libpaddle.so"
14+
"${PADDLE_PATH}/libs/libcommon.so"
15+
# "${PADDLE_PATH}/libs/libphi.so" # currently libphi.so is static linked, we need remove it when it's shared linked
16+
"${PADDLE_PATH}/libs/libphi_core.so"
17+
)
18+
set(
19+
TORCH_INSTALL_PREFIX
20+
"${PADDLE_PATH}"
21+
)
22+
## Debug: print Torch-related CMake variables so we can see what was found
23+
message(STATUS "[Torch DEBUG] Torch_DIR=${Torch_DIR}")
24+
message(STATUS "[Torch DEBUG] TORCH_LIBRARIES=${TORCH_LIBRARIES}")
25+
message(STATUS "[Torch DEBUG] TORCH_CXX_FLAGS=${TORCH_CXX_FLAGS}")
26+
message(STATUS "[Torch DEBUG] TORCH_INSTALL_PREFIX=${TORCH_INSTALL_PREFIX}")
27+
message(STATUS "[Torch DEBUG] TORCH_VERSION=${TORCH_VERSION}")
28+
if(TARGET "torch")
29+
message(STATUS "[Torch DEBUG] Imported target 'torch' exists")
30+
endif()
931
find_package(Python3 ${PYTHON_VERSION} EXACT COMPONENTS Development)
1032

1133
if(DEFINED TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR AND TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR)
@@ -15,7 +37,7 @@ else()
1537
# TODO set warnings as errors on Windows as well.
1638
# set(TORCHCODEC_WERROR_OPTION "/WX")
1739
else()
18-
set(TORCHCODEC_WERROR_OPTION "-Werror")
40+
#set(TORCHCODEC_WERROR_OPTION "-Werror")
1941
endif()
2042
endif()
2143

@@ -47,6 +69,9 @@ function(make_torchcodec_sublibrary
4769
PRIVATE
4870
./../../../
4971
"${TORCH_INSTALL_PREFIX}/include"
72+
"${TORCH_INSTALL_PREFIX}/include/third_party"
73+
"${TORCH_INSTALL_PREFIX}/include/paddle/phi/api/include/compat"
74+
"${TORCH_INSTALL_PREFIX}/include/paddle/phi/api/include/compat/torch/csrc/api/include"
5075
${Python3_INCLUDE_DIRS}
5176
)
5277

@@ -57,6 +82,7 @@ function(make_torchcodec_sublibrary
5782
${library_name}
5883
PUBLIC
5984
${library_dependencies}
85+
"-Wl,--as-needed"
6086
)
6187

6288
endfunction()

0 commit comments

Comments
 (0)