diff --git a/pyproject.toml b/pyproject.toml index 265b7e73..195b0324 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ keywords = [ "scraping", ] dependencies = [ - "apify-client>=2.3.0,<3.0.0", + "apify-client @ git+https://github.com/apify/apify-client-python.git@typed-clients", "apify-shared>=2.0.0,<3.0.0", "crawlee>=1.0.4,<2.0.0", "cachetools>=5.5.0", diff --git a/src/apify/_actor.py b/src/apify/_actor.py index fac2ea8b..664947fa 100644 --- a/src/apify/_actor.py +++ b/src/apify/_actor.py @@ -885,7 +885,8 @@ async def start( f'Invalid timeout {timeout!r}: expected `None`, `"inherit"`, `"RemainingTime"`, or a `timedelta`.' ) - api_result = await client.actor(actor_id).start( + actor_client = client.actor(actor_id) + run = await actor_client.start( run_input=run_input, content_type=content_type, build=build, @@ -895,7 +896,11 @@ async def start( webhooks=serialized_webhooks, ) - return ActorRun.model_validate(api_result) + if run is None: + raise RuntimeError(f'Failed to start Actor with ID "{actor_id}".') + + run_dict = run.model_dump(by_alias=True) + return ActorRun.model_validate(run_dict) async def abort( self, @@ -923,13 +928,18 @@ async def abort( self._raise_if_not_initialized() client = self.new_client(token=token) if token else self.apify_client + run_client = client.run(run_id) if status_message: - await client.run(run_id).update(status_message=status_message) + await run_client.update(status_message=status_message) + + run = await run_client.abort(gracefully=gracefully) - api_result = await client.run(run_id).abort(gracefully=gracefully) + if run is None: + raise RuntimeError(f'Failed to abort Actor run with ID "{run_id}".') - return ActorRun.model_validate(api_result) + run_dict = run.model_dump(by_alias=True) + return ActorRun.model_validate(run_dict) async def call( self, @@ -1002,7 +1012,8 @@ async def call( f'Invalid timeout {timeout!r}: expected `None`, `"inherit"`, `"RemainingTime"`, or a `timedelta`.' ) - api_result = await client.actor(actor_id).call( + actor_client = client.actor(actor_id) + run = await actor_client.call( run_input=run_input, content_type=content_type, build=build, @@ -1013,7 +1024,11 @@ async def call( logger=logger, ) - return ActorRun.model_validate(api_result) + if run is None: + raise RuntimeError(f'Failed to call Actor with ID "{actor_id}".') + + run_dict = run.model_dump(by_alias=True) + return ActorRun.model_validate(run_dict) async def call_task( self, @@ -1075,7 +1090,8 @@ async def call_task( else: raise ValueError(f'Invalid timeout {timeout!r}: expected `None`, `"inherit"`, or a `timedelta`.') - api_result = await client.task(task_id).call( + task_client = client.task(task_id) + run = await task_client.call( task_input=task_input, build=build, memory_mbytes=memory_mbytes, @@ -1084,7 +1100,11 @@ async def call_task( wait_secs=int(wait.total_seconds()) if wait is not None else None, ) - return ActorRun.model_validate(api_result) + if run is None: + raise RuntimeError(f'Failed to call Task with ID "{task_id}".') + + run_dict = run.model_dump(by_alias=True) + return ActorRun.model_validate(run_dict) async def metamorph( self, @@ -1261,11 +1281,19 @@ async def set_status_message( if not self.configuration.actor_run_id: raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.') - api_result = await self.apify_client.run(self.configuration.actor_run_id).update( - status_message=status_message, is_status_message_terminal=is_terminal + run_client = self.apify_client.run(self.configuration.actor_run_id) + run = await run_client.update( + status_message=status_message, + is_status_message_terminal=is_terminal, ) - return ActorRun.model_validate(api_result) + if run is None: + raise RuntimeError( + f'Failed to set status message for Actor run with ID "{self.configuration.actor_run_id}".' + ) + + run_dict = run.model_dump(by_alias=True) + return ActorRun.model_validate(run_dict) async def create_proxy_configuration( self, diff --git a/src/apify/_charging.py b/src/apify/_charging.py index f8f8ddb2..97040474 100644 --- a/src/apify/_charging.py +++ b/src/apify/_charging.py @@ -351,14 +351,21 @@ async def _fetch_pricing_info(self) -> _FetchedPricingInfoDict: if self._actor_run_id is None: raise RuntimeError('Actor run ID not found even though the Actor is running on Apify') - run = run_validator.validate_python(await self._client.run(self._actor_run_id).get()) + run = await self._client.run(self._actor_run_id).get() + if run is None: raise RuntimeError('Actor run not found') + run_dict = run.model_dump(by_alias=True) + actor_run = run_validator.validate_python(run_dict) + + if actor_run is None: + raise RuntimeError('Actor run not found') + return _FetchedPricingInfoDict( - pricing_info=run.pricing_info, - charged_event_counts=run.charged_event_counts or {}, - max_total_charge_usd=run.options.max_total_charge_usd or Decimal('inf'), + pricing_info=actor_run.pricing_info, + charged_event_counts=actor_run.charged_event_counts or {}, + max_total_charge_usd=actor_run.options.max_total_charge_usd or Decimal('inf'), ) # Local development without environment variables diff --git a/src/apify/_models.py b/src/apify/_models.py index 6dd72dca..7519e3a7 100644 --- a/src/apify/_models.py +++ b/src/apify/_models.py @@ -94,32 +94,82 @@ class ActorRunUsage(BaseModel): @docs_group('Actor') class ActorRun(BaseModel): + """Represents an Actor run and its associated data.""" + __model_config__ = ConfigDict(populate_by_name=True) id: Annotated[str, Field(alias='id')] + """Unique identifier of the Actor run.""" + act_id: Annotated[str, Field(alias='actId')] + """ID of the Actor that was run.""" + user_id: Annotated[str, Field(alias='userId')] + """ID of the user who started the run.""" + actor_task_id: Annotated[str | None, Field(alias='actorTaskId')] = None + """ID of the Actor task, if the run was started from a task.""" + started_at: Annotated[datetime, Field(alias='startedAt')] + """Time when the Actor run started.""" + finished_at: Annotated[datetime | None, Field(alias='finishedAt')] = None + """Time when the Actor run finished.""" + status: Annotated[ActorJobStatus, Field(alias='status')] + """Current status of the Actor run.""" + status_message: Annotated[str | None, Field(alias='statusMessage')] = None + """Detailed message about the run status.""" + is_status_message_terminal: Annotated[bool | None, Field(alias='isStatusMessageTerminal')] = None + """Whether the status message is terminal (final).""" + meta: Annotated[ActorRunMeta, Field(alias='meta')] + """Metadata about the Actor run.""" + stats: Annotated[ActorRunStats, Field(alias='stats')] + """Statistics of the Actor run.""" + options: Annotated[ActorRunOptions, Field(alias='options')] + """Configuration options for the Actor run.""" + build_id: Annotated[str, Field(alias='buildId')] + """ID of the Actor build used for this run.""" + exit_code: Annotated[int | None, Field(alias='exitCode')] = None + """Exit code of the Actor run process.""" + default_key_value_store_id: Annotated[str, Field(alias='defaultKeyValueStoreId')] + """ID of the default key-value store for this run.""" + default_dataset_id: Annotated[str, Field(alias='defaultDatasetId')] + """ID of the default dataset for this run.""" + default_request_queue_id: Annotated[str, Field(alias='defaultRequestQueueId')] + """ID of the default request queue for this run.""" + build_number: Annotated[str | None, Field(alias='buildNumber')] = None + """Build number of the Actor build used for this run.""" + container_url: Annotated[str, Field(alias='containerUrl')] + """URL of the container running the Actor.""" + is_container_server_ready: Annotated[bool | None, Field(alias='isContainerServerReady')] = None + """Whether the container's HTTP server is ready to accept requests.""" + git_branch_name: Annotated[str | None, Field(alias='gitBranchName')] = None + """Name of the git branch used for the Actor build.""" + usage: Annotated[ActorRunUsage | None, Field(alias='usage')] = None + """Resource usage statistics for the run.""" + usage_total_usd: Annotated[float | None, Field(alias='usageTotalUsd')] = None + """Total cost of the run in USD.""" + usage_usd: Annotated[ActorRunUsage | None, Field(alias='usageUsd')] = None + """Resource usage costs in USD.""" + pricing_info: Annotated[ FreeActorPricingInfo | FlatPricePerMonthActorPricingInfo @@ -128,10 +178,13 @@ class ActorRun(BaseModel): | None, Field(alias='pricingInfo', discriminator='pricing_model'), ] = None + """Pricing information for the Actor.""" + charged_event_counts: Annotated[ dict[str, int] | None, Field(alias='chargedEventCounts'), ] = None + """Count of charged events for pay-per-event pricing model.""" class FreeActorPricingInfo(BaseModel): diff --git a/src/apify/storage_clients/_apify/_alias_resolving.py b/src/apify/storage_clients/_apify/_alias_resolving.py index e357333f..ea4ecaa6 100644 --- a/src/apify/storage_clients/_apify/_alias_resolving.py +++ b/src/apify/storage_clients/_apify/_alias_resolving.py @@ -14,7 +14,7 @@ from collections.abc import Callable from types import TracebackType - from apify_client.clients import ( + from apify_client._resource_clients import ( DatasetClientAsync, DatasetCollectionClientAsync, KeyValueStoreClientAsync, @@ -105,8 +105,8 @@ async def open_by_alias( # Create new unnamed storage and store alias mapping raw_metadata = await collection_client.get_or_create() - await alias_resolver.store_mapping(storage_id=raw_metadata['id']) - return get_resource_client_by_id(raw_metadata['id']) + await alias_resolver.store_mapping(storage_id=raw_metadata.id) + return get_resource_client_by_id(raw_metadata.id) class AliasResolver: diff --git a/src/apify/storage_clients/_apify/_api_client_creation.py b/src/apify/storage_clients/_apify/_api_client_creation.py index 39e2a087..864e1a6d 100644 --- a/src/apify/storage_clients/_apify/_api_client_creation.py +++ b/src/apify/storage_clients/_apify/_api_client_creation.py @@ -8,7 +8,7 @@ from apify.storage_clients._apify._alias_resolving import open_by_alias if TYPE_CHECKING: - from apify_client.clients import DatasetClientAsync, KeyValueStoreClientAsync, RequestQueueClientAsync + from apify_client._resource_clients import DatasetClientAsync, KeyValueStoreClientAsync, RequestQueueClientAsync from apify._configuration import Configuration @@ -137,13 +137,13 @@ def get_resource_client(storage_id: str) -> DatasetClientAsync: # Default storage does not exist. Create a new one. if not raw_metadata: raw_metadata = await collection_client.get_or_create() - resource_client = get_resource_client(raw_metadata['id']) + resource_client = get_resource_client(raw_metadata.id) return resource_client # Open by name. case (None, str(), None, _): raw_metadata = await collection_client.get_or_create(name=name) - return get_resource_client(raw_metadata['id']) + return get_resource_client(raw_metadata.id) # Open by ID. case (None, None, str(), _): diff --git a/src/apify/storage_clients/_apify/_dataset_client.py b/src/apify/storage_clients/_apify/_dataset_client.py index a918bddd..ba015e23 100644 --- a/src/apify/storage_clients/_apify/_dataset_client.py +++ b/src/apify/storage_clients/_apify/_dataset_client.py @@ -2,6 +2,7 @@ import asyncio import warnings +from datetime import datetime from logging import getLogger from typing import TYPE_CHECKING, Any @@ -17,7 +18,7 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator - from apify_client.clients import DatasetClientAsync + from apify_client._resource_clients import DatasetClientAsync from crawlee._types import JsonSerializable from apify import Configuration @@ -65,7 +66,18 @@ def __init__( @override async def get_metadata(self) -> DatasetMetadata: metadata = await self._api_client.get() - return DatasetMetadata.model_validate(metadata) + + if metadata is None: + raise ValueError('Failed to retrieve dataset metadata.') + + return DatasetMetadata( + id=metadata.id, + name=metadata.name, + created_at=datetime.fromisoformat(metadata.created_at.replace('Z', '+00:00')), + modified_at=datetime.fromisoformat(metadata.modified_at.replace('Z', '+00:00')), + accessed_at=datetime.fromisoformat(metadata.accessed_at.replace('Z', '+00:00')), + item_count=int(metadata.item_count), + ) @classmethod async def open( diff --git a/src/apify/storage_clients/_apify/_key_value_store_client.py b/src/apify/storage_clients/_apify/_key_value_store_client.py index b422b464..f47626e5 100644 --- a/src/apify/storage_clients/_apify/_key_value_store_client.py +++ b/src/apify/storage_clients/_apify/_key_value_store_client.py @@ -2,6 +2,7 @@ import asyncio import warnings +from datetime import datetime from logging import getLogger from typing import TYPE_CHECKING, Any @@ -11,12 +12,12 @@ from crawlee.storage_clients.models import KeyValueStoreRecord, KeyValueStoreRecordMetadata from ._api_client_creation import create_storage_api_client -from ._models import ApifyKeyValueStoreMetadata, KeyValueStoreListKeysPage +from ._models import ApifyKeyValueStoreMetadata if TYPE_CHECKING: from collections.abc import AsyncIterator - from apify_client.clients import KeyValueStoreClientAsync + from apify_client._resource_clients import KeyValueStoreClientAsync from apify import Configuration @@ -54,7 +55,18 @@ def __init__( @override async def get_metadata(self) -> ApifyKeyValueStoreMetadata: metadata = await self._api_client.get() - return ApifyKeyValueStoreMetadata.model_validate(metadata) + + if metadata is None: + raise ValueError('Failed to retrieve dataset metadata.') + + return ApifyKeyValueStoreMetadata( + id=metadata.id, + name=metadata.name, + created_at=datetime.fromisoformat(metadata.created_at.replace('Z', '+00:00')), + modified_at=datetime.fromisoformat(metadata.modified_at.replace('Z', '+00:00')), + accessed_at=datetime.fromisoformat(metadata.accessed_at.replace('Z', '+00:00')), + url_signing_secret_key=metadata.url_signing_secret_key, + ) @classmethod async def open( @@ -143,14 +155,13 @@ async def iterate_keys( count = 0 while True: - response = await self._api_client.list_keys(exclusive_start_key=exclusive_start_key) - list_key_page = KeyValueStoreListKeysPage.model_validate(response) + list_key_page = await self._api_client.list_keys(exclusive_start_key=exclusive_start_key) for item in list_key_page.items: # Convert KeyValueStoreKeyInfo to KeyValueStoreRecordMetadata record_metadata = KeyValueStoreRecordMetadata( key=item.key, - size=item.size, + size=int(item.size), content_type='application/octet-stream', # Content type not available from list_keys ) yield record_metadata diff --git a/src/apify/storage_clients/_apify/_request_queue_client.py b/src/apify/storage_clients/_apify/_request_queue_client.py index 9a589ec1..4dc5a6d9 100644 --- a/src/apify/storage_clients/_apify/_request_queue_client.py +++ b/src/apify/storage_clients/_apify/_request_queue_client.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime from logging import getLogger from typing import TYPE_CHECKING, Final, Literal @@ -15,7 +16,7 @@ if TYPE_CHECKING: from collections.abc import Sequence - from apify_client.clients import RequestQueueClientAsync + from apify_client._resource_clients import RequestQueueClientAsync from crawlee import Request from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata @@ -77,26 +78,34 @@ async def get_metadata(self) -> ApifyRequestQueueMetadata: Returns: Request queue metadata with accurate counts and timestamps, combining API data with local estimates. """ - response = await self._api_client.get() + metadata = await self._api_client.get() - if response is None: + if metadata is None: raise ValueError('Failed to fetch request queue metadata from the API.') + total_request_count = int(metadata.total_request_count) + handled_request_count = int(metadata.handled_request_count) + pending_request_count = int(metadata.pending_request_count) + created_at = datetime.fromisoformat(metadata.created_at.replace('Z', '+00:00')) + modified_at = datetime.fromisoformat(metadata.modified_at.replace('Z', '+00:00')) + accessed_at = datetime.fromisoformat(metadata.accessed_at.replace('Z', '+00:00')) + # Enhance API response with local estimations to account for propagation delays (API data can be delayed # by a few seconds, while local estimates are immediately accurate). return ApifyRequestQueueMetadata( - id=response['id'], - name=response['name'], - total_request_count=max(response['totalRequestCount'], self._implementation.metadata.total_request_count), - handled_request_count=max( - response['handledRequestCount'], self._implementation.metadata.handled_request_count + id=metadata.id, + name=metadata.name, + total_request_count=max(total_request_count, self._implementation.metadata.total_request_count), + handled_request_count=max(handled_request_count, self._implementation.metadata.handled_request_count), + pending_request_count=pending_request_count, + created_at=min(created_at, self._implementation.metadata.created_at), + modified_at=max(modified_at, self._implementation.metadata.modified_at), + accessed_at=max(accessed_at, self._implementation.metadata.accessed_at), + had_multiple_clients=metadata.had_multiple_clients or self._implementation.metadata.had_multiple_clients, + stats=RequestQueueStats.model_validate( + metadata.stats.model_dump(by_alias=True) if metadata.stats else {}, + by_alias=True, ), - pending_request_count=response['pendingRequestCount'], - created_at=min(response['createdAt'], self._implementation.metadata.created_at), - modified_at=max(response['modifiedAt'], self._implementation.metadata.modified_at), - accessed_at=max(response['accessedAt'], self._implementation.metadata.accessed_at), - had_multiple_clients=response['hadMultipleClients'] or self._implementation.metadata.had_multiple_clients, - stats=RequestQueueStats.model_validate(response['stats'], by_alias=True), ) @classmethod @@ -145,7 +154,7 @@ async def open( raw_metadata = await api_client.get() if raw_metadata is None: raise ValueError('Failed to retrieve request queue metadata from the API.') - metadata = ApifyRequestQueueMetadata.model_validate(raw_metadata) + metadata = ApifyRequestQueueMetadata.model_validate(raw_metadata.model_dump(by_alias=True)) return cls( api_client=api_client, diff --git a/src/apify/storage_clients/_apify/_request_queue_shared_client.py b/src/apify/storage_clients/_apify/_request_queue_shared_client.py index 4a00e8bc..ab456267 100644 --- a/src/apify/storage_clients/_apify/_request_queue_shared_client.py +++ b/src/apify/storage_clients/_apify/_request_queue_shared_client.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Coroutine, Sequence - from apify_client.clients import RequestQueueClientAsync + from apify_client._resource_clients import RequestQueueClientAsync logger = getLogger(__name__) @@ -121,18 +121,17 @@ async def add_batch_of_requests( if new_requests: # Prepare requests for API by converting to dictionaries. - requests_dict = [ - request.model_dump( - by_alias=True, - ) - for request in new_requests - ] + requests_dict = [request.model_dump(by_alias=True) for request in new_requests] # Send requests to API. - api_response = AddRequestsResponse.model_validate( - await self._api_client.batch_add_requests(requests=requests_dict, forefront=forefront) + batch_response = await self._api_client.batch_add_requests( + requests=requests_dict, + forefront=forefront, ) + batch_response_dict = batch_response.model_dump(by_alias=True) + api_response = AddRequestsResponse.model_validate(batch_response_dict) + # Add the locally known already present processed requests based on the local cache. api_response.processed_requests.extend(already_present_requests) @@ -312,7 +311,8 @@ async def _get_request_by_id(self, request_id: str) -> Request | None: if response is None: return None - return Request.model_validate(response) + response_dict = response.model_dump(by_alias=True) + return Request.model_validate(response_dict) async def _ensure_head_is_non_empty(self) -> None: """Ensure that the queue head has requests if they are available in the queue.""" @@ -388,7 +388,7 @@ async def _update_request( ) return ProcessedRequest.model_validate( - {'uniqueKey': request.unique_key} | response, + {'uniqueKey': request.unique_key} | response.model_dump(by_alias=True), ) async def _list_head( @@ -431,19 +431,19 @@ async def _list_head( self._should_check_for_forefront_requests = False # Otherwise fetch from API - response = await self._api_client.list_and_lock_head( + list_and_lost_data = await self._api_client.list_and_lock_head( lock_secs=int(self._DEFAULT_LOCK_TIME.total_seconds()), limit=limit, ) # Update the queue head cache - self._queue_has_locked_requests = response.get('queueHasLockedRequests', False) + self._queue_has_locked_requests = list_and_lost_data.queue_has_locked_requests # Check if there is another client working with the RequestQueue - self.metadata.had_multiple_clients = response.get('hadMultipleClients', False) + self.metadata.had_multiple_clients = list_and_lost_data.had_multiple_clients - for request_data in response.get('items', []): - request = Request.model_validate(request_data) - request_id = request_data.get('id') + for request_data in list_and_lost_data.items: + request = Request.model_validate(request_data.model_dump(by_alias=True)) + request_id = request_data.id # Skip requests without ID or unique key if not request.unique_key or not request_id: @@ -473,7 +473,8 @@ async def _list_head( # After adding new requests to the forefront, any existing leftover locked request is kept in the end. self._queue_head.append(leftover_id) - return RequestQueueHead.model_validate(response) + list_and_lost_dict = list_and_lost_data.model_dump(by_alias=True) + return RequestQueueHead.model_validate(list_and_lost_dict) def _cache_request( self, diff --git a/src/apify/storage_clients/_apify/_request_queue_single_client.py b/src/apify/storage_clients/_apify/_request_queue_single_client.py index 7cc202bb..f6e07a75 100644 --- a/src/apify/storage_clients/_apify/_request_queue_single_client.py +++ b/src/apify/storage_clients/_apify/_request_queue_single_client.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from collections.abc import Sequence - from apify_client.clients import RequestQueueClientAsync + from apify_client._resource_clients import RequestQueueClientAsync logger = getLogger(__name__) @@ -147,22 +147,20 @@ async def add_batch_of_requests( if new_requests: # Prepare requests for API by converting to dictionaries. - requests_dict = [ - request.model_dump( - by_alias=True, - ) - for request in new_requests - ] + requests_dict = [request.model_dump(by_alias=True) for request in new_requests] # Send requests to API. - api_response = AddRequestsResponse.model_validate( - await self._api_client.batch_add_requests(requests=requests_dict, forefront=forefront) - ) + batch_response = await self._api_client.batch_add_requests(requests=requests_dict, forefront=forefront) + batch_response_dict = batch_response.model_dump(by_alias=True) + api_response = AddRequestsResponse.model_validate(batch_response_dict) + # Add the locally known already present processed requests based on the local cache. api_response.processed_requests.extend(already_present_requests) + # Remove unprocessed requests from the cache for unprocessed_request in api_response.unprocessed_requests: - self._requests_cache.pop(unique_key_to_request_id(unprocessed_request.unique_key), None) + request_id = unique_key_to_request_id(unprocessed_request.unique_key) + self._requests_cache.pop(request_id, None) else: api_response = AddRequestsResponse( @@ -288,16 +286,17 @@ async def _list_head(self) -> None: # Update metadata # Check if there is another client working with the RequestQueue - self.metadata.had_multiple_clients = response.get('hadMultipleClients', False) + self.metadata.had_multiple_clients = response.had_multiple_clients # Should warn once? This might be outside expected context if the other consumers consumes at the same time - if modified_at := response.get('queueModifiedAt'): + if response.queue_modified_at: + modified_at = datetime.fromisoformat(response.queue_modified_at.replace('Z', '+00:00')) self.metadata.modified_at = max(self.metadata.modified_at, modified_at) # Update the cached data - for request_data in response.get('items', []): - request = Request.model_validate(request_data) - request_id = request_data['id'] + for request_data in response.items: + request = Request.model_validate(request_data.model_dump(by_alias=True)) + request_id = request_data.id if request_id in self._requests_in_progress: # Ignore requests that are already in progress, we will not process them again. @@ -328,7 +327,8 @@ async def _get_request_by_id(self, id: str) -> Request | None: if response is None: return None - request = Request.model_validate(response) + response_dict = response.model_dump(by_alias=True) + request = Request.model_validate(response_dict) # Updated local caches if id in self._requests_in_progress: @@ -365,7 +365,7 @@ async def _update_request( ) return ProcessedRequest.model_validate( - {'uniqueKey': request.unique_key} | response, + {'uniqueKey': request.unique_key} | response.model_dump(by_alias=True), ) async def _init_caches(self) -> None: @@ -378,9 +378,9 @@ async def _init_caches(self) -> None: Local deduplication is cheaper, it takes 1 API call for whole cache and 1 read operation per request. """ response = await self._api_client.list_requests(limit=10_000) - for request_data in response.get('items', []): - request = Request.model_validate(request_data) - request_id = request_data['id'] + for request_data in response.items: + request = Request.model_validate(request_data.model_dump(by_alias=True)) + request_id = request_data.id if request.was_already_handled: # Cache just id for deduplication diff --git a/tests/integration/actor/conftest.py b/tests/integration/actor/conftest.py index e7376853..e3bb5e84 100644 --- a/tests/integration/actor/conftest.py +++ b/tests/integration/actor/conftest.py @@ -22,7 +22,7 @@ from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping from decimal import Decimal - from apify_client.clients.resource_clients import ActorClientAsync + from apify_client._resource_clients import ActorClientAsync _TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN' _API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL' @@ -236,19 +236,19 @@ async def _make_actor( ], ) - actor_client = client.actor(created_actor['id']) + actor_client = client.actor(created_actor.id) print(f'Building Actor {actor_name}...') build_result = await actor_client.build(version_number='0.0') - build_client = client.build(build_result['id']) + build_client = client.build(build_result.id) build_client_result = await build_client.wait_for_finish(wait_secs=600) assert build_client_result is not None - assert build_client_result['status'] == ActorJobStatus.SUCCEEDED + assert build_client_result.status == ActorJobStatus.SUCCEEDED # We only mark the client for cleanup if the build succeeded, so that if something goes wrong here, # you have a chance to check the error. - actors_for_cleanup.append(created_actor['id']) + actors_for_cleanup.append(created_actor.id) return actor_client @@ -256,17 +256,15 @@ async def _make_actor( # Delete all the generated Actors. for actor_id in actors_for_cleanup: - actor_client = ApifyClient(token=apify_token, api_url=os.getenv(_API_URL_ENV_VAR)).actor(actor_id) - - if (actor := actor_client.get()) is not None: - actor_client.update( - pricing_infos=[ - *actor.get('pricingInfos', []), - { - 'pricingModel': 'FREE', - }, - ] - ) + apify_client = ApifyClient(token=apify_token, api_url=os.getenv(_API_URL_ENV_VAR)) + actor_client = apify_client.actor(actor_id) + actor = actor_client.get() + + if actor is not None and actor.pricing_infos is not None: + # Convert Pydantic models to dicts before mixing with plain dict + existing_pricing_infos = [pi.model_dump(by_alias=True, exclude_none=True) for pi in actor.pricing_infos] + new_pricing_infos = [*existing_pricing_infos, {'pricingModel': 'FREE'}] + actor_client.update(pricing_infos=new_pricing_infos) actor_client.delete() @@ -306,17 +304,16 @@ async def _run_actor( run_input: Any = None, max_total_charge_usd: Decimal | None = None, ) -> ActorRun: - call_result = await actor.call( - run_input=run_input, - max_total_charge_usd=max_total_charge_usd, - ) + call_result = await actor.call(run_input=run_input, max_total_charge_usd=max_total_charge_usd) + + assert call_result is not None, 'Failed to start Actor run: missing run ID in the response.' - assert isinstance(call_result, dict), 'The result of ActorClientAsync.call() is not a dictionary.' - assert 'id' in call_result, 'The result of ActorClientAsync.call() does not contain an ID.' + run_client = apify_client_async.run(call_result.id) + actor_run = await run_client.wait_for_finish(wait_secs=600) - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + assert actor_run is not None, 'Actor run did not finish successfully within the expected time.' - return ActorRun.model_validate(run_result) + actor_run_dict = actor_run.model_dump(by_alias=True) + return ActorRun.model_validate(actor_run_dict) return _run_actor diff --git a/tests/integration/actor/test_actor_api_helpers.py b/tests/integration/actor/test_actor_api_helpers.py index 68579fc3..0d807db0 100644 --- a/tests/integration/actor/test_actor_api_helpers.py +++ b/tests/integration/actor/test_actor_api_helpers.py @@ -130,12 +130,15 @@ async def main_outer() -> None: inner_run_status = await Actor.apify_client.actor(inner_actor_id).last_run().get() assert inner_run_status is not None - assert inner_run_status.get('status') in ['READY', 'RUNNING'] + assert inner_run_status.status in ['READY', 'RUNNING'] inner_actor = await make_actor(label='start-inner', main_func=main_inner) outer_actor = await make_actor(label='start-outer', main_func=main_outer) - inner_actor_id = (await inner_actor.get() or {})['id'] + inner_actor_get_result = await inner_actor.get() + assert inner_actor_get_result is not None, 'Failed to get inner actor ID' + + inner_actor_id = inner_actor_get_result.id test_value = crypto_random_object_id() run_result_outer = await run_actor( @@ -175,12 +178,15 @@ async def main_outer() -> None: inner_run_status = await Actor.apify_client.actor(inner_actor_id).last_run().get() assert inner_run_status is not None - assert inner_run_status.get('status') == 'SUCCEEDED' + assert inner_run_status.status == 'SUCCEEDED' inner_actor = await make_actor(label='call-inner', main_func=main_inner) outer_actor = await make_actor(label='call-outer', main_func=main_outer) - inner_actor_id = (await inner_actor.get() or {})['id'] + inner_actor_get_result = await inner_actor.get() + assert inner_actor_get_result is not None, 'Failed to get inner actor ID' + + inner_actor_id = inner_actor_get_result.id test_value = crypto_random_object_id() run_result_outer = await run_actor( @@ -221,12 +227,15 @@ async def main_outer() -> None: inner_run_status = await Actor.apify_client.task(inner_task_id).last_run().get() assert inner_run_status is not None - assert inner_run_status.get('status') == 'SUCCEEDED' + assert inner_run_status.status == 'SUCCEEDED' inner_actor = await make_actor(label='call-task-inner', main_func=main_inner) outer_actor = await make_actor(label='call-task-outer', main_func=main_outer) - inner_actor_id = (await inner_actor.get() or {})['id'] + inner_actor_get_result = await inner_actor.get() + assert inner_actor_get_result is not None, 'Failed to get inner actor ID' + + inner_actor_id = inner_actor_get_result.id test_value = crypto_random_object_id() task = await apify_client_async.tasks().create( @@ -237,7 +246,7 @@ async def main_outer() -> None: run_result_outer = await run_actor( outer_actor, - run_input={'test_value': test_value, 'inner_task_id': task['id']}, + run_input={'test_value': test_value, 'inner_task_id': task.id}, ) assert run_result_outer.status == 'SUCCEEDED' @@ -248,7 +257,7 @@ async def main_outer() -> None: assert inner_output_record is not None assert inner_output_record['value'] == f'{test_value}_XXX_{test_value}' - await apify_client_async.task(task['id']).delete() + await apify_client_async.task(task.id).delete() @pytest.mark.skip(reason='Requires Actor permissions beyond limited permissions, see #715.') @@ -274,7 +283,8 @@ async def main_outer() -> None: inner_actor = await make_actor(label='abort-inner', main_func=main_inner) outer_actor = await make_actor(label='abort-outer', main_func=main_outer) - inner_run_id = (await inner_actor.start())['id'] + actor_run = await inner_actor.start() + inner_run_id = actor_run.id run_result_outer = await run_actor( outer_actor, @@ -283,9 +293,14 @@ async def main_outer() -> None: assert run_result_outer.status == 'SUCCEEDED' - await inner_actor.last_run().wait_for_finish(wait_secs=600) - inner_actor_last_run_dict = await inner_actor.last_run().get() - inner_actor_last_run = ActorRun.model_validate(inner_actor_last_run_dict) + inner_actor_run_client = inner_actor.last_run() + inner_actor_run = await inner_actor_run_client.wait_for_finish(wait_secs=600) + + if inner_actor_run is None: + raise AssertionError('Failed to get inner actor run after aborting it.') + + inner_actor_run_dict = inner_actor_run.model_dump(by_alias=True) + inner_actor_last_run = ActorRun.model_validate(inner_actor_run_dict) assert inner_actor_last_run.status == 'ABORTED' @@ -331,7 +346,10 @@ async def main_outer() -> None: inner_actor = await make_actor(label='metamorph-inner', main_func=main_inner) outer_actor = await make_actor(label='metamorph-outer', main_func=main_outer) - inner_actor_id = (await inner_actor.get() or {})['id'] + inner_actor_get_result = await inner_actor.get() + assert inner_actor_get_result is not None, 'Failed to get inner actor ID' + + inner_actor_id = inner_actor_get_result.id test_value = crypto_random_object_id() run_result_outer = await run_actor( @@ -444,7 +462,7 @@ async def main_client() -> None: ) server_actor_run = await server_actor.start() - server_actor_container_url = server_actor_run['containerUrl'] + server_actor_container_url = server_actor_run.container_url server_actor_initialized = await server_actor.last_run().key_value_store().get_record('INITIALIZED') while not server_actor_initialized: @@ -458,8 +476,14 @@ async def main_client() -> None: assert ac_run_result.status == 'SUCCEEDED' - sa_run_result_dict = await server_actor.last_run().wait_for_finish(wait_secs=600) - sa_run_result = ActorRun.model_validate(sa_run_result_dict) + sa_run_client = server_actor.last_run() + sa_run_client_run = await sa_run_client.wait_for_finish(wait_secs=600) + + if sa_run_client_run is None: + raise AssertionError('Failed to get server actor run after waiting for finish.') + + sa_run_client_run_dict = sa_run_client_run.model_dump(by_alias=True) + sa_run_result = ActorRun.model_validate(sa_run_client_run_dict) assert sa_run_result.status == 'SUCCEEDED' diff --git a/tests/integration/actor/test_actor_charge.py b/tests/integration/actor/test_actor_charge.py index d72062bc..0f1ee3cf 100644 --- a/tests/integration/actor/test_actor_charge.py +++ b/tests/integration/actor/test_actor_charge.py @@ -15,7 +15,7 @@ from collections.abc import Iterable from apify_client import ApifyClientAsync - from apify_client.clients import ActorClientAsync + from apify_client._resource_clients import ActorClientAsync from .conftest import MakeActorFunction, RunActorFunction @@ -48,13 +48,13 @@ async def main() -> None: }, }, }, - ] + ], ) actor = await actor_client.get() assert actor is not None - return str(actor['id']) + return str(actor.id) @pytest_asyncio.fixture(scope='function', loop_scope='module') @@ -82,8 +82,13 @@ async def test_actor_charge_basic( # Refetch until the platform gets its act together for is_last_attempt, _ in retry_counter(30): await asyncio.sleep(1) - updated_run = await apify_client_async.run(run.id).get() - run = ActorRun.model_validate(updated_run) + + run_client = apify_client_async.run(run.id) + updated_run = await run_client.get() + assert updated_run is not None, 'Updated run should not be None' + + updated_run_dict = updated_run.model_dump(by_alias=True) + run = ActorRun.model_validate(updated_run_dict) try: assert run.status == ActorJobStatus.SUCCEEDED @@ -104,8 +109,13 @@ async def test_actor_charge_limit( # Refetch until the platform gets its act together for is_last_attempt, _ in retry_counter(30): await asyncio.sleep(1) - updated_run = await apify_client_async.run(run.id).get() - run = ActorRun.model_validate(updated_run) + + run_client = apify_client_async.run(run.id) + updated_run = await run_client.get() + assert updated_run is not None, 'Updated run should not be None' + + updated_run_dict = updated_run.model_dump(by_alias=True) + run = ActorRun.model_validate(updated_run_dict) try: assert run.status == ActorJobStatus.SUCCEEDED diff --git a/tests/integration/actor/test_actor_dataset.py b/tests/integration/actor/test_actor_dataset.py index 409df584..0c265d99 100644 --- a/tests/integration/actor/test_actor_dataset.py +++ b/tests/integration/actor/test_actor_dataset.py @@ -139,7 +139,7 @@ async def test_force_cloud( try: dataset_details = await dataset_client.get() assert dataset_details is not None - assert dataset_details.get('name') == dataset_name + assert dataset_details.name == dataset_name dataset_items = await dataset_client.list_items() assert dataset_items.items == [dataset_item] diff --git a/tests/integration/actor/test_actor_key_value_store.py b/tests/integration/actor/test_actor_key_value_store.py index 2ed9af29..7cfa2a2c 100644 --- a/tests/integration/actor/test_actor_key_value_store.py +++ b/tests/integration/actor/test_actor_key_value_store.py @@ -79,7 +79,7 @@ async def test_force_cloud( try: key_value_store_details = await key_value_store_client.get() assert key_value_store_details is not None - assert key_value_store_details.get('name') == key_value_store_name + assert key_value_store_details.name == key_value_store_name key_value_store_record = await key_value_store_client.get_record('foo') assert key_value_store_record is not None @@ -141,7 +141,7 @@ async def main_get() -> None: default_kvs_info = await actor_set.last_run().key_value_store().get() assert default_kvs_info is not None - run_result_get = await run_actor(actor_get, run_input={'kvs-id': default_kvs_info['id']}) + run_result_get = await run_actor(actor_get, run_input={'kvs-id': default_kvs_info.id}) assert run_result_get.status == 'SUCCEEDED' diff --git a/tests/integration/actor/test_actor_lifecycle.py b/tests/integration/actor/test_actor_lifecycle.py index e2f98e6d..a9d45533 100644 --- a/tests/integration/actor/test_actor_lifecycle.py +++ b/tests/integration/actor/test_actor_lifecycle.py @@ -136,8 +136,8 @@ async def main() -> None: requests = ['https://example.com/1', 'https://example.com/2'] run = await Actor.apify_client.run(Actor.configuration.actor_run_id or '').get() - assert run - first_run = run.get('stats', {}).get('rebootCount', 0) == 0 + assert run is not None + first_run = run.stats.reboot_count == 0 @crawler.router.default_handler async def default_handler(context: BasicCrawlingContext) -> None: diff --git a/tests/integration/actor/test_actor_request_queue.py b/tests/integration/actor/test_actor_request_queue.py index 1cc4c543..c956a57d 100644 --- a/tests/integration/actor/test_actor_request_queue.py +++ b/tests/integration/actor/test_actor_request_queue.py @@ -84,8 +84,8 @@ async def main() -> None: # Get raw client, because stats are not exposed in `RequestQueue` class, but are available in raw client rq_client = Actor.apify_client.request_queue(request_queue_id=rq.id) _rq = await rq_client.get() - assert _rq - stats_before = _rq.get('stats', {}) + assert _rq is not None + stats_before = _rq.stats Actor.log.info(stats_before) # Add same request twice @@ -94,11 +94,16 @@ async def main() -> None: await asyncio.sleep(10) # Wait to be sure that metadata are updated _rq = await rq_client.get() - assert _rq - stats_after = _rq.get('stats', {}) + assert _rq is not None + stats_after = _rq.stats Actor.log.info(stats_after) - assert (stats_after['writeCount'] - stats_before['writeCount']) == 1 + assert stats_after is not None + assert stats_after.write_count is not None + assert stats_before is not None + assert stats_before.write_count is not None + + assert (stats_after.write_count - stats_before.write_count) == 1 actor = await make_actor(label='rq-deduplication', main_func=main) run_result = await run_actor(actor) @@ -133,8 +138,8 @@ async def main() -> None: # Get raw client, because stats are not exposed in `RequestQueue` class, but are available in raw client rq_client = Actor.apify_client.request_queue(request_queue_id=rq.id) _rq = await rq_client.get() - assert _rq - stats_before = _rq.get('stats', {}) + assert _rq is not None + stats_before = _rq.stats Actor.log.info(stats_before) # Add same request twice @@ -143,11 +148,16 @@ async def main() -> None: await asyncio.sleep(10) # Wait to be sure that metadata are updated _rq = await rq_client.get() - assert _rq - stats_after = _rq.get('stats', {}) + assert _rq is not None + stats_after = _rq.stats Actor.log.info(stats_after) - assert (stats_after['writeCount'] - stats_before['writeCount']) == 2 + assert stats_after is not None + assert stats_after.write_count is not None + assert stats_before is not None + assert stats_before.write_count is not None + + assert (stats_after.write_count - stats_before.write_count) == 2 actor = await make_actor(label='rq-deduplication', main_func=main) run_result = await run_actor(actor) @@ -189,10 +199,13 @@ async def main() -> None: # Get raw client, because stats are not exposed in `RequestQueue` class, but are available in raw client rq_client = Actor.apify_client.request_queue(request_queue_id=rq.id) _rq = await rq_client.get() - assert _rq - stats_before = _rq.get('stats', {}) + assert _rq is not None + stats_before = _rq.stats Actor.log.info(stats_before) + assert stats_before is not None + assert stats_before.write_count is not None + # Add batches of some new and some already present requests in workers async def add_requests_worker() -> None: await rq.add_requests(requests[: next(batch_size)]) @@ -203,11 +216,14 @@ async def add_requests_worker() -> None: await asyncio.sleep(10) # Wait to be sure that metadata are updated _rq = await rq_client.get() - assert _rq - stats_after = _rq.get('stats', {}) + assert _rq is not None + stats_after = _rq.stats Actor.log.info(stats_after) - assert (stats_after['writeCount'] - stats_before['writeCount']) == len(requests) + assert stats_after is not None + assert stats_after.write_count is not None + + assert (stats_after.write_count - stats_before.write_count) == len(requests) actor = await make_actor(label='rq-parallel-deduplication', main_func=main) run_result = await run_actor(actor) @@ -290,8 +306,15 @@ async def main() -> None: # Redirect logs even from the resurrected run streamed_log = await run_client.get_streamed_log(from_start=False) await run_client.resurrect() + async with streamed_log: - run_result = ActorRun.model_validate(await run_client.wait_for_finish(wait_secs=600)) + run = await run_client.wait_for_finish(wait_secs=600) + + if run is None: + raise AssertionError('Failed to get resurrected run.') + + run_dict = run.model_dump(by_alias=True) + run_result = ActorRun.model_validate(run_dict) assert run_result.status == 'SUCCEEDED' diff --git a/tests/integration/apify_api/test_request_queue.py b/tests/integration/apify_api/test_request_queue.py index e90c1600..98583a47 100644 --- a/tests/integration/apify_api/test_request_queue.py +++ b/tests/integration/apify_api/test_request_queue.py @@ -8,6 +8,7 @@ import pytest +from apify_client._models import Data13, UnprocessedRequest from apify_shared.consts import ApifyEnvVars from crawlee import service_locator from crawlee.crawlers import BasicCrawler @@ -930,7 +931,7 @@ async def test_request_queue_had_multiple_clients( # Check that it is correctly in the API api_response = await api_client.get() assert api_response - assert api_response['hadMultipleClients'] is True + assert api_response.had_multiple_clients is True async def test_request_queue_not_had_multiple_clients( @@ -949,7 +950,7 @@ async def test_request_queue_not_had_multiple_clients( api_client = apify_client_async.request_queue(request_queue_id=rq.id) api_response = await api_client.get() assert api_response - assert api_response['hadMultipleClients'] is False + assert api_response.had_multiple_clients is False async def test_request_queue_simple_and_full_at_the_same_time( @@ -1122,11 +1123,11 @@ async def test_force_cloud( request_queue_details = await request_queue_client.get() assert request_queue_details is not None - assert request_queue_details.get('name') == request_queue_apify.name + assert request_queue_details.name == request_queue_apify.name request_queue_request = await request_queue_client.get_request(request_info.id) assert request_queue_request is not None - assert request_queue_request['url'] == 'http://example.com' + assert request_queue_request.url == 'http://example.com' async def test_request_queue_is_finished( @@ -1161,22 +1162,31 @@ async def test_request_queue_deduplication_unprocessed_requests( # Get raw client, because stats are not exposed in `RequestQueue` class, but are available in raw client rq_client = Actor.apify_client.request_queue(request_queue_id=request_queue_apify.id) _rq = await rq_client.get() - assert _rq - stats_before = _rq.get('stats', {}) + assert _rq is not None + stats_before = _rq.stats Actor.log.info(stats_before) - def return_unprocessed_requests(requests: list[dict], *_: Any, **__: Any) -> dict[str, list[dict]]: + assert stats_before is not None + assert stats_before.write_count is not None + + def return_unprocessed_requests(requests: list[dict], *_: Any, **__: Any) -> Data13: """Simulate API returning unprocessed requests.""" - return { - 'processedRequests': [], - 'unprocessedRequests': [ - {'url': request['url'], 'uniqueKey': request['uniqueKey'], 'method': request['method']} - for request in requests - ], - } + unprocessed_requests = [ + UnprocessedRequest.model_construct( + url=request['url'], + unique_key=request['uniqueKey'], + method=request['method'], + ) + for request in requests + ] + + return Data13.model_construct( + processed_requests=[], + unprocessed_requests=unprocessed_requests, + ) with mock.patch( - 'apify_client.clients.resource_clients.request_queue.RequestQueueClientAsync.batch_add_requests', + 'apify_client._resource_clients.request_queue.RequestQueueClientAsync.batch_add_requests', side_effect=return_unprocessed_requests, ): # Simulate failed API call for adding requests. Request was not processed and should not be cached. @@ -1187,8 +1197,11 @@ def return_unprocessed_requests(requests: list[dict], *_: Any, **__: Any) -> dic await asyncio.sleep(10) # Wait to be sure that metadata are updated _rq = await rq_client.get() - assert _rq - stats_after = _rq.get('stats', {}) + assert _rq is not None + stats_after = _rq.stats Actor.log.info(stats_after) - assert (stats_after['writeCount'] - stats_before['writeCount']) == 1 + assert stats_after is not None + assert stats_after.write_count is not None + + assert (stats_after.write_count - stats_before.write_count) == 1 diff --git a/tests/unit/actor/test_actor_helpers.py b/tests/unit/actor/test_actor_helpers.py index f71cd44c..c10378d3 100644 --- a/tests/unit/actor/test_actor_helpers.py +++ b/tests/unit/actor/test_actor_helpers.py @@ -5,6 +5,7 @@ import pytest from apify_client import ApifyClientAsync +from apify_client._models import Run from apify_shared.consts import ApifyEnvVars, WebhookEventType from apify import Actor, Webhook @@ -15,45 +16,48 @@ @pytest.fixture -def fake_actor_run() -> dict: - return { - 'id': 'asdfasdf', - 'buildId': '3ads35', - 'buildNumber': '3.4.5', - 'actId': 'actor_id', - 'actorId': 'actor_id', - 'userId': 'user_id', - 'startedAt': '2024-08-08 12:12:44', - 'status': 'RUNNING', - 'meta': {'origin': 'API'}, - 'containerUrl': 'http://0.0.0.0:3333', - 'defaultDatasetId': 'dhasdrfughaerguoi', - 'defaultKeyValueStoreId': 'asjkldhguiofg', - 'defaultRequestQueueId': 'lkjgklserjghios', - 'stats': { - 'inputBodyLen': 0, - 'restartCount': 0, - 'resurrectCount': 0, - 'memAvgBytes': 0, - 'memMaxBytes': 0, - 'memCurrentBytes': 0, - 'cpuAvgUsage': 0, - 'cpuMaxUsage': 0, - 'cpuCurrentUsage': 0, - 'netRxBytes': 0, - 'netTxBytes': 0, - 'durationMillis': 3333, - 'runTimeSecs': 33, - 'metamorph': 0, - 'computeUnits': 4.33, - }, - 'options': { - 'build': '', - 'timeoutSecs': 44, - 'memoryMbytes': 4096, - 'diskMbytes': 16384, - }, - } +def fake_actor_run() -> Run: + return Run.model_validate( + { + 'id': 'asdfasdf', + 'buildId': '3ads35', + 'buildNumber': '3.4.5', + 'actId': 'actor_id', + 'actorId': 'actor_id', + 'userId': 'user_id', + 'startedAt': '2024-08-08 12:12:44', + 'status': 'RUNNING', + 'meta': {'origin': 'API'}, + 'containerUrl': 'http://0.0.0.0:3333', + 'defaultDatasetId': 'dhasdrfughaerguoi', + 'defaultKeyValueStoreId': 'asjkldhguiofg', + 'defaultRequestQueueId': 'lkjgklserjghios', + 'generalAccess': 'RESTRICTED', + 'stats': { + 'inputBodyLen': 0, + 'restartCount': 0, + 'resurrectCount': 0, + 'memAvgBytes': 0, + 'memMaxBytes': 0, + 'memCurrentBytes': 0, + 'cpuAvgUsage': 0, + 'cpuMaxUsage': 0, + 'cpuCurrentUsage': 0, + 'netRxBytes': 0, + 'netTxBytes': 0, + 'durationMillis': 3333, + 'runTimeSecs': 33, + 'metamorph': 0, + 'computeUnits': 4.33, + }, + 'options': { + 'build': '', + 'timeoutSecs': 44, + 'memoryMbytes': 4096, + 'diskMbytes': 16384, + }, + } + ) async def test_new_client_config_creation(monkeypatch: pytest.MonkeyPatch) -> None: @@ -74,7 +78,7 @@ async def test_new_client_config_creation(monkeypatch: pytest.MonkeyPatch) -> No await my_actor.exit() -async def test_call_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: dict) -> None: +async def test_call_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: Run) -> None: apify_client_async_patcher.patch('actor', 'call', return_value=fake_actor_run) actor_id = 'some-actor-id' @@ -86,7 +90,7 @@ async def test_call_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, f assert apify_client_async_patcher.calls['actor']['call'][0][0][0].resource_id == actor_id -async def test_call_actor_task(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: dict) -> None: +async def test_call_actor_task(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: Run) -> None: apify_client_async_patcher.patch('task', 'call', return_value=fake_actor_run) task_id = 'some-task-id' @@ -97,7 +101,7 @@ async def test_call_actor_task(apify_client_async_patcher: ApifyClientAsyncPatch assert apify_client_async_patcher.calls['task']['call'][0][0][0].resource_id == task_id -async def test_start_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: dict) -> None: +async def test_start_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: Run) -> None: apify_client_async_patcher.patch('actor', 'start', return_value=fake_actor_run) actor_id = 'some-id' @@ -108,7 +112,7 @@ async def test_start_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, assert apify_client_async_patcher.calls['actor']['start'][0][0][0].resource_id == actor_id -async def test_abort_actor_run(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: dict) -> None: +async def test_abort_actor_run(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: Run) -> None: apify_client_async_patcher.patch('run', 'abort', return_value=fake_actor_run) run_id = 'some-run-id' diff --git a/tests/unit/actor/test_actor_lifecycle.py b/tests/unit/actor/test_actor_lifecycle.py index 0d5b9277..3954bf59 100644 --- a/tests/unit/actor/test_actor_lifecycle.py +++ b/tests/unit/actor/test_actor_lifecycle.py @@ -12,6 +12,7 @@ import pytest import websockets.asyncio.server +from apify_client._models import Run from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars from crawlee.events._types import Event, EventPersistStateData @@ -238,31 +239,34 @@ async def handler(websocket: websockets.asyncio.server.ServerConnection) -> None mock_run_client = Mock() mock_run_client.run.return_value.get = AsyncMock( - side_effect=lambda: { - 'id': 'asdf', - 'actId': 'asdf', - 'userId': 'adsf', - 'startedAt': datetime.now(timezone.utc), - 'status': 'RUNNING', - 'meta': {'origin': 'DEVELOPMENT'}, - 'stats': { - 'inputBodyLen': 99, - 'restartCount': 0, - 'resurrectCount': 0, - 'computeUnits': 1, - }, - 'options': { - 'build': 'asdf', - 'timeoutSecs': 4, - 'memoryMbytes': 1024, - 'diskMbytes': 1024, - }, - 'buildId': 'hjkl', - 'defaultDatasetId': 'hjkl', - 'defaultKeyValueStoreId': 'hjkl', - 'defaultRequestQueueId': 'hjkl', - 'containerUrl': 'https://hjkl', - } + side_effect=lambda: Run.model_validate( + { + 'id': 'asdf', + 'actId': 'asdf', + 'userId': 'adsf', + 'startedAt': datetime.now(timezone.utc).isoformat(), + 'status': 'RUNNING', + 'meta': {'origin': 'DEVELOPMENT'}, + 'buildId': 'hjkl', + 'defaultDatasetId': 'hjkl', + 'defaultKeyValueStoreId': 'hjkl', + 'defaultRequestQueueId': 'hjkl', + 'containerUrl': 'https://hjkl', + 'buildNumber': '0.0.1', + 'generalAccess': 'RESTRICTED', + 'stats': { + 'restartCount': 0, + 'resurrectCount': 0, + 'computeUnits': 1, + }, + 'options': { + 'build': 'asdf', + 'timeoutSecs': 4, + 'memoryMbytes': 1024, + 'diskMbytes': 1024, + }, + } + ) ) with mock.patch.object(Actor, 'new_client', return_value=mock_run_client): diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 3f792ad8..94f1f5f2 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -62,6 +62,7 @@ def prepare_test_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Callabl def _prepare_test_env() -> None: if hasattr(apify._actor.Actor, '__wrapped__'): delattr(apify._actor.Actor, '__wrapped__') + apify._actor.Actor._is_initialized = False # Set the environment variable for the local storage directory to the temporary path. diff --git a/uv.lock b/uv.lock index 602929ff..7d96d787 100644 --- a/uv.lock +++ b/uv.lock @@ -72,7 +72,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "apify-client", specifier = ">=2.3.0,<3.0.0" }, + { name = "apify-client", git = "https://github.com/apify/apify-client-python.git?rev=typed-clients" }, { name = "apify-shared", specifier = ">=2.0.0,<3.0.0" }, { name = "cachetools", specifier = ">=5.5.0" }, { name = "crawlee", specifier = ">=1.0.4,<2.0.0" }, @@ -111,18 +111,14 @@ dev = [ [[package]] name = "apify-client" -version = "2.3.0" -source = { registry = "https://pypi.org/simple" } +version = "2.3.1" +source = { git = "https://github.com/apify/apify-client-python.git?rev=typed-clients#d591e7ac294e320e5dc09eebb943daa727548689" } dependencies = [ { name = "apify-shared" }, { name = "colorama" }, { name = "impit" }, { name = "more-itertools" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/dd/e3/08c1eb269d4559e5c01343347c913423c24efd425f0c2bd9f743e28c8a86/apify_client-2.3.0.tar.gz", hash = "sha256:ff6d32e27d5205343e89057ac0e0c02b53a9219ccedfd30a3c4d70d13d931488", size = 389101, upload-time = "2025-11-13T13:42:33.923Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/b6/6dabe41804932e020557450abc8bc74725a942c6f08969021efe965d4260/apify_client-2.3.0-py3-none-any.whl", hash = "sha256:6ae9b1461c2a15ab19c6131bfcab55be9362259cced9b254b827b4c3b6c12d40", size = 85996, upload-time = "2025-11-13T13:42:32.012Z" }, -] [[package]] name = "apify-shared"