pitrou commented on code in PR #37769: URL: https://github.com/apache/arrow/pull/37769#discussion_r1329621605
########## dev/archery/archery/integration/tester.py: ########## @@ -17,12 +17,180 @@ # Base class for language-specific integration test harnesses +from abc import ABC, abstractmethod +import os import subprocess +import typing from .util import log -class Tester(object): +_Predicate = typing.Callable[[], bool] + + +class CDataExporter(ABC): + + @abstractmethod + def export_schema_from_json(self, json_path: os.PathLike, + c_schema_ptr: object): + """ + Read a JSON integration file and export its schema. + + Parameters + ---------- + json_path : Path + Path to the JSON file + c_schema_ptr : cffi pointer value + Pointer to the ``ArrowSchema`` struct to export to. + """ + + @abstractmethod + def export_batch_from_json(self, json_path: os.PathLike, + num_batch: int, + c_array_ptr: object): + """ + Read a JSON integration file and export one of its batches. + + Parameters + ---------- + json_path : Path + Path to the JSON file + num_batch : int + Number of the record batch in the JSON file + c_schema_ptr : cffi pointer value + Pointer to the ``ArrowArray`` struct to export to. + """ + + @property + @abstractmethod + def supports_releasing_memory(self) -> bool: + """ + Whether the implementation is able to release memory deterministically. + + Here, "release memory" means that, after the `release` callback of + a C Data Interface export is called, `compare_allocation_state` is + able to trigger the deallocation of the memory underlying the export + (for example buffer data). + + If false, then `record_allocation_state` and `compare_allocation_state` + are allowed to raise NotImplementedError. + """ + + def record_allocation_state(self) -> object: + """ + Record the current memory allocation state. + + Returns + ------- + state : object + Opaque object representing the allocation state, + for example the number of allocated bytes. + """ + raise NotImplementedError + + def compare_allocation_state(self, recorded: object, + gc_until: typing.Callable[[_Predicate], bool] + ) -> bool: + """ + Compare the current memory allocation state with the recorded one. + + Parameters + ---------- + recorded : object + The previous allocation state returned by + `record_allocation_state()` + gc_until : callable + A callable itself accepting a callable predicate, and + returning a boolean. + `gc_until` should try to release memory until the predicate + becomes true, or until it decides to give up. The final value + of the predicate should be returned. + `gc_until` is typically provided by the C Data Interface importer. + + Returns + ------- + success : bool + Whether memory allocation state finally reached its previously + recorded value. + """ + raise NotImplementedError + + +class CDataImporter(ABC): + + @abstractmethod + def import_schema_and_compare_to_json(self, json_path: os.PathLike, + c_schema_ptr: object): + """ + Import schema and compare it to the schema of a JSON integration file. + + An error is raised if importing fails or the schemas differ. + + Parameters + ---------- + json_path : Path + The path to the JSON file + c_schema_ptr : cffi pointer value + Pointer to the ``ArrowSchema`` struct to import from. + """ + + @abstractmethod + def import_batch_and_compare_to_json(self, json_path: os.PathLike, + num_batch: int, + c_array_ptr: object): + """ + Import record batch and compare it to one of the batches + from a JSON integration file. + + The schema used for importing the record batch is the one from + the JSON file. + + An error is raised if importing fails or the batches differ. + + Parameters + ---------- + json_path : Path + The path to the JSON file + num_batch : int + Number of the record batch in the JSON file + c_array_ptr : cffi pointer value + Pointer to the ``ArrowArray`` struct to import from. + """ + + @property + @abstractmethod + def supports_releasing_memory(self) -> bool: + """ + Whether the implementation is able to release memory deterministically. + + Here, "release memory" means calling the `release` callback of + a C Data Interface export (which should then trigger a deallocation + mechanism on the exporter). + + If false, then `gc_until` is allowed to raise NotImplementedError. + """ + + def gc_until(self, predicate: _Predicate): Review Comment: Actually, it should be an abstract method. Thanks for noticing. ########## dev/archery/archery/integration/tester.py: ########## @@ -17,12 +17,180 @@ # Base class for language-specific integration test harnesses +from abc import ABC, abstractmethod +import os import subprocess +import typing from .util import log -class Tester(object): +_Predicate = typing.Callable[[], bool] + + +class CDataExporter(ABC): + + @abstractmethod + def export_schema_from_json(self, json_path: os.PathLike, + c_schema_ptr: object): + """ + Read a JSON integration file and export its schema. + + Parameters + ---------- + json_path : Path + Path to the JSON file + c_schema_ptr : cffi pointer value + Pointer to the ``ArrowSchema`` struct to export to. + """ + + @abstractmethod + def export_batch_from_json(self, json_path: os.PathLike, + num_batch: int, + c_array_ptr: object): + """ + Read a JSON integration file and export one of its batches. + + Parameters + ---------- + json_path : Path + Path to the JSON file + num_batch : int + Number of the record batch in the JSON file + c_schema_ptr : cffi pointer value + Pointer to the ``ArrowArray`` struct to export to. + """ + + @property + @abstractmethod + def supports_releasing_memory(self) -> bool: + """ + Whether the implementation is able to release memory deterministically. + + Here, "release memory" means that, after the `release` callback of + a C Data Interface export is called, `compare_allocation_state` is + able to trigger the deallocation of the memory underlying the export + (for example buffer data). + + If false, then `record_allocation_state` and `compare_allocation_state` + are allowed to raise NotImplementedError. + """ + + def record_allocation_state(self) -> object: + """ + Record the current memory allocation state. + + Returns + ------- + state : object + Opaque object representing the allocation state, + for example the number of allocated bytes. + """ + raise NotImplementedError + + def compare_allocation_state(self, recorded: object, + gc_until: typing.Callable[[_Predicate], bool] + ) -> bool: + """ + Compare the current memory allocation state with the recorded one. + + Parameters + ---------- + recorded : object + The previous allocation state returned by + `record_allocation_state()` + gc_until : callable + A callable itself accepting a callable predicate, and + returning a boolean. + `gc_until` should try to release memory until the predicate + becomes true, or until it decides to give up. The final value + of the predicate should be returned. + `gc_until` is typically provided by the C Data Interface importer. + + Returns + ------- + success : bool + Whether memory allocation state finally reached its previously + recorded value. + """ + raise NotImplementedError + + +class CDataImporter(ABC): + + @abstractmethod + def import_schema_and_compare_to_json(self, json_path: os.PathLike, + c_schema_ptr: object): + """ + Import schema and compare it to the schema of a JSON integration file. + + An error is raised if importing fails or the schemas differ. + + Parameters + ---------- + json_path : Path + The path to the JSON file + c_schema_ptr : cffi pointer value + Pointer to the ``ArrowSchema`` struct to import from. + """ + + @abstractmethod + def import_batch_and_compare_to_json(self, json_path: os.PathLike, + num_batch: int, + c_array_ptr: object): + """ + Import record batch and compare it to one of the batches + from a JSON integration file. + + The schema used for importing the record batch is the one from + the JSON file. + + An error is raised if importing fails or the batches differ. + + Parameters + ---------- + json_path : Path + The path to the JSON file + num_batch : int + Number of the record batch in the JSON file + c_array_ptr : cffi pointer value + Pointer to the ``ArrowArray`` struct to import from. + """ + + @property + @abstractmethod + def supports_releasing_memory(self) -> bool: + """ + Whether the implementation is able to release memory deterministically. + + Here, "release memory" means calling the `release` callback of + a C Data Interface export (which should then trigger a deallocation + mechanism on the exporter). + + If false, then `gc_until` is allowed to raise NotImplementedError. + """ + + def gc_until(self, predicate: _Predicate): Review Comment: Yes, you're right. Let me do this. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
