Skip to content

Commit 3b74abf

Browse files
author
amiya
committed
first commit
0 parents  commit 3b74abf

39 files changed

Lines changed: 11574 additions & 0 deletions

‎.github/workflows/CI.yml‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
# This file is autogenerated by maturin v1.10.2
2+
# To update, run
3+
#
4+
# maturin generate-ci github
5+
#
6+
name: CI
7+
8+
on:
9+
push:
10+
branches:
11+
- main
12+
- master
13+
tags:
14+
- '*'
15+
pull_request:
16+
workflow_dispatch:
17+
18+
permissions:
19+
contents: read
20+
21+
jobs:
22+
linux:
23+
runs-on: ${{ matrix.platform.runner }}
24+
strategy:
25+
matrix:
26+
platform:
27+
- runner: ubuntu-22.04
28+
target: x86_64
29+
- runner: ubuntu-22.04
30+
target: x86
31+
- runner: ubuntu-22.04
32+
target: aarch64
33+
- runner: ubuntu-22.04
34+
target: armv7
35+
- runner: ubuntu-22.04
36+
target: s390x
37+
- runner: ubuntu-22.04
38+
target: ppc64le
39+
steps:
40+
- uses: actions/checkout@v4
41+
- uses: actions/setup-python@v5
42+
with:
43+
python-version: 3.x
44+
- name: Build wheels
45+
uses: PyO3/maturin-action@v1
46+
with:
47+
target: ${{ matrix.platform.target }}
48+
args: --release --out dist --find-interpreter
49+
sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
50+
manylinux: auto
51+
- name: Upload wheels
52+
uses: actions/upload-artifact@v4
53+
with:
54+
name: wheels-linux-${{ matrix.platform.target }}
55+
path: dist
56+
57+
musllinux:
58+
runs-on: ${{ matrix.platform.runner }}
59+
strategy:
60+
matrix:
61+
platform:
62+
- runner: ubuntu-22.04
63+
target: x86_64
64+
- runner: ubuntu-22.04
65+
target: x86
66+
- runner: ubuntu-22.04
67+
target: aarch64
68+
- runner: ubuntu-22.04
69+
target: armv7
70+
steps:
71+
- uses: actions/checkout@v4
72+
- uses: actions/setup-python@v5
73+
with:
74+
python-version: 3.x
75+
- name: Build wheels
76+
uses: PyO3/maturin-action@v1
77+
with:
78+
target: ${{ matrix.platform.target }}
79+
args: --release --out dist --find-interpreter
80+
sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
81+
manylinux: musllinux_1_2
82+
- name: Upload wheels
83+
uses: actions/upload-artifact@v4
84+
with:
85+
name: wheels-musllinux-${{ matrix.platform.target }}
86+
path: dist
87+
88+
windows:
89+
runs-on: ${{ matrix.platform.runner }}
90+
strategy:
91+
matrix:
92+
platform:
93+
- runner: windows-latest
94+
target: x64
95+
- runner: windows-latest
96+
target: x86
97+
steps:
98+
- uses: actions/checkout@v4
99+
- uses: actions/setup-python@v5
100+
with:
101+
python-version: 3.x
102+
architecture: ${{ matrix.platform.target }}
103+
- name: Build wheels
104+
uses: PyO3/maturin-action@v1
105+
with:
106+
target: ${{ matrix.platform.target }}
107+
args: --release --out dist --find-interpreter
108+
sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
109+
- name: Upload wheels
110+
uses: actions/upload-artifact@v4
111+
with:
112+
name: wheels-windows-${{ matrix.platform.target }}
113+
path: dist
114+
115+
macos:
116+
runs-on: ${{ matrix.platform.runner }}
117+
strategy:
118+
matrix:
119+
platform:
120+
- runner: macos-13
121+
target: x86_64
122+
- runner: macos-14
123+
target: aarch64
124+
steps:
125+
- uses: actions/checkout@v4
126+
- uses: actions/setup-python@v5
127+
with:
128+
python-version: 3.x
129+
- name: Build wheels
130+
uses: PyO3/maturin-action@v1
131+
with:
132+
target: ${{ matrix.platform.target }}
133+
args: --release --out dist --find-interpreter
134+
sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
135+
- name: Upload wheels
136+
uses: actions/upload-artifact@v4
137+
with:
138+
name: wheels-macos-${{ matrix.platform.target }}
139+
path: dist
140+
141+
sdist:
142+
runs-on: ubuntu-latest
143+
steps:
144+
- uses: actions/checkout@v4
145+
- name: Build sdist
146+
uses: PyO3/maturin-action@v1
147+
with:
148+
command: sdist
149+
args: --out dist
150+
- name: Upload sdist
151+
uses: actions/upload-artifact@v4
152+
with:
153+
name: wheels-sdist
154+
path: dist
155+
156+
release:
157+
name: Release
158+
runs-on: ubuntu-latest
159+
if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }}
160+
needs: [linux, musllinux, windows, macos, sdist]
161+
permissions:
162+
# Use to sign the release artifacts
163+
id-token: write
164+
# Used to upload release artifacts
165+
contents: write
166+
# Used to generate artifact attestation
167+
attestations: write
168+
steps:
169+
- uses: actions/download-artifact@v4
170+
- name: Generate artifact attestation
171+
uses: actions/attest-build-provenance@v2
172+
with:
173+
subject-path: 'wheels-*/*'
174+
- name: Publish to PyPI
175+
if: ${{ startsWith(github.ref, 'refs/tags/') }}
176+
uses: PyO3/maturin-action@v1
177+
env:
178+
MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
179+
with:
180+
command: upload
181+
args: --non-interactive --skip-existing wheels-*/*

‎.gitignore‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# Python
2+
__pycache__/
3+
*.py[cod]
4+
*$py.class
5+
*.so
6+
.Python
7+
build/
8+
develop-eggs/
9+
dist/
10+
downloads/
11+
eggs/
12+
.eggs/
13+
lib/
14+
lib64/
15+
parts/
16+
sdist/
17+
var/
18+
wheels/
19+
*.egg-info/
20+
.installed.cfg
21+
*.egg
22+
23+
# Virtual environments
24+
.venv/
25+
venv/
26+
ENV/
27+
env/
28+
29+
# Rust
30+
target/
31+
Cargo.lock
32+
**/*.rs.bk
33+
*.pdb
34+
35+
# PyO3 / Maturin
36+
*.so
37+
*.dylib
38+
*.dll
39+
*.pyd
40+
41+
# IDEs
42+
.vscode/
43+
.idea/
44+
*.swp
45+
*.swo
46+
*~
47+
.DS_Store
48+
49+
# Testing
50+
.pytest_cache/
51+
.coverage
52+
htmlcov/
53+
.tox/
54+
55+
# Jupyter Notebook
56+
.ipynb_checkpoints
57+
58+
# Distribution
59+
*.whl
60+
61+
.claude/

‎.python-version‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3.13

‎CHANGELOG.md‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
# Changelog
2+
3+
## [Unreleased] - 2025-11-30
4+
5+
### Added
6+
7+
#### 1. Thread Pool Configuration
8+
- Added `configure_thread_pool(num_threads, stack_size)` function to configure the global Rayon thread pool
9+
- Added `get_thread_pool_info()` function to query current thread pool configuration
10+
- Thread pool can be configured with custom number of threads and stack size
11+
- Provides better resource management for parallel operations
12+
13+
#### 2. Priority Queue System
14+
- Added `@parallel_priority` decorator for priority-based task scheduling
15+
- Tasks execute based on priority value (higher = more important)
16+
- Implemented with BinaryHeap for O(log n) operations
17+
- Added `start_priority_worker()` and `stop_priority_worker()` functions
18+
- Worker thread automatically starts when using `@parallel_priority`
19+
20+
#### 3. Enhanced Task Cancellation
21+
- Added `cancel_with_timeout(timeout_secs)` method to AsyncHandle
22+
- Gracefully cancel tasks with a timeout
23+
- Returns boolean indicating success
24+
- Added `is_cancelled()` method to check cancellation status
25+
- Added `elapsed_time()` method to track task duration
26+
- Added `get_name()` method to retrieve function name
27+
- Improved cancellation with atomic boolean flags
28+
29+
#### 4. Performance Profiling Tools
30+
- Added `@profiled` decorator for automatic performance tracking
31+
- All `@parallel` tasks are now automatically profiled
32+
- Added `PerformanceMetrics` class with:
33+
- `total_tasks`: Total number of executions
34+
- `completed_tasks`: Successful executions
35+
- `failed_tasks`: Failed executions
36+
- `total_execution_time_ms`: Total time in milliseconds
37+
- `average_execution_time_ms`: Average time per execution
38+
- Added `get_metrics(name)` to retrieve metrics for specific function
39+
- Added `get_all_metrics()` to get all collected metrics
40+
- Added `reset_metrics()` to clear all metrics
41+
- Global counters for total tasks, completed, and failed
42+
- Thread-safe implementation using atomic operations and DashMap
43+
44+
### Technical Implementation
45+
46+
#### New Dependencies
47+
- Uses existing dependencies (no new external dependencies required)
48+
- Leverages `once_cell::Lazy` for global state
49+
- Uses `std::sync::atomic` for lock-free counters
50+
- Uses `std::collections::BinaryHeap` for priority queue
51+
52+
#### Architecture Changes
53+
- Added global thread pool configuration with `Lazy<Arc<Mutex<Option<rayon::ThreadPool>>>>`
54+
- Priority queue worker runs in background thread
55+
- Metrics collected in lock-free DashMap
56+
- Cancellation tokens using `Arc<AtomicBool>`
57+
- All parallel tasks now track execution time and success/failure
58+
59+
### Documentation
60+
- Added comprehensive `docs/NEW_FEATURES.md` with:
61+
- API documentation for all new features
62+
- Usage examples
63+
- Best practices
64+
- Troubleshooting guide
65+
- Migration guide
66+
- Updated main README.md with new features section
67+
- Added example scripts:
68+
- `examples/test_new_features.py`: Comprehensive test of all features
69+
- `examples/quick_test_features.py`: Quick feature validation
70+
- `examples/basic_test.py`: API availability check
71+
72+
### Testing
73+
- All existing tests continue to pass
74+
- New features validated with test scripts
75+
- Backward compatible with existing code
76+
77+
### Performance Impact
78+
- Thread pool configuration: One-time setup cost
79+
- Priority queue: ~10-50μs overhead per task
80+
- Profiling: ~1-5μs overhead per task (minimal)
81+
- Cancellation: No overhead unless cancelled
82+
- All features use lock-free data structures where possible
83+
84+
### API Summary
85+
86+
**Thread Pool:**
87+
```python
88+
mp.configure_thread_pool(num_threads=8)
89+
mp.get_thread_pool_info()
90+
```
91+
92+
**Priority Queue:**
93+
```python
94+
@mp.parallel_priority
95+
def task(data):
96+
pass
97+
98+
handle = task(data, priority=100)
99+
```
100+
101+
**Cancellation:**
102+
```python
103+
handle.cancel_with_timeout(2.0)
104+
handle.is_cancelled()
105+
handle.elapsed_time()
106+
handle.get_name()
107+
```
108+
109+
**Profiling:**
110+
```python
111+
@mp.profiled
112+
def func():
113+
pass
114+
115+
mp.get_metrics("func")
116+
mp.get_all_metrics()
117+
mp.reset_metrics()
118+
```
119+
120+
## [0.1.0] - Previous
121+
122+
### Initial Release
123+
- Basic decorators: @timer, @log_calls, @CallCounter, @retry, @memoize
124+
- Parallel execution: @parallel, @parallel_fast, @parallel_pool
125+
- Optimized implementations with Crossbeam and Rayon
126+
- AsyncHandle for task management
127+
- True GIL-free parallelism with Rust threads

‎Cargo.toml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
[package]
2+
name = "makeParallel"
3+
version = "0.1.0"
4+
edition = "2024"
5+
6+
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
7+
[lib]
8+
name = "makeParallel"
9+
crate-type = ["cdylib"]
10+
11+
[dependencies]
12+
pyo3 = { version = "0.27.1", features = ["extension-module"] }
13+
crossbeam = "0.8"
14+
rayon = "1.10"
15+
dashmap = "6.1"
16+
once_cell = "1.20"

0 commit comments

Comments
 (0)