Skip to content

cache

This module provides mechanisms to enable pyotb raster caching on the local filesystem.

Cache

Bases: Input

Enable to manage a given pipeline output, depending on if it's already in the cache.

Source code in scenes/cache.py 
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
class Cache(pyotb.Input):
    """
    Enable to manage a given pipeline output, depending on if it's already in
    the cache.
    """

    def __init__(
            self,
            pyotb_output: pyotb.Output,
            temporary_directory: str = None,
            output_parameter_key: str = None,
            extension: str = None,
            pixel_type: str = None,
            summary_modifier=None
    ):
        """
        Initialize the cache.

        Args:
            pyotb_output: a pyotb.Output instance.
            temporary_directory: a temporary directory for the cached files.
                Default is system temp directory.
            output_parameter_key: output parameter key (default is first key)
            extension: file extension (default: .tif)
            pixel_type: pixel type
            summary_modifier: optional function to modify the summary
        """
        # Get app
        pyotb_app = pyotb_output.parent_pyotb_app

        # Get summary
        assert isinstance(pyotb_app, pyotb.core.App)
        summary = pyotb.summarize(pyotb_app)  # need pyotb >= 1.5.1

        # Modify summary
        if summary_modifier:
            summary = summary_modifier(summary)

        # Summary --> md5sum
        desc = json.dumps(summary)
        md5sum = hashlib.md5(desc.encode('utf-8')).hexdigest()

        # App name
        app_name = summary["name"]

        # Cache filename
        if not output_parameter_key:
            output_parameter_key = pyotb_app.output_image_key
        if not extension:
            extension = ".tif?&gdal:co:COMPRESS=DEFLATE&gdal:co:BIGTIFF=YES"
        elif not extension.startswith("."):
            extension = f".{extension}"
        if not pixel_type:
            pixel_type = "float"
        if not temporary_directory:
            temporary_directory = tempfile.gettempdir()
        if not os.path.exists(temporary_directory):
            os.makedirs(temporary_directory, exist_ok=True)
        prefix = os.path.join(
            temporary_directory, f"{app_name}_{output_parameter_key}_{md5sum}"
        )
        cache_file = f"{prefix}{extension}"
        json_file = f"{prefix}.json"

        # Check which cache files already exist
        if not os.path.exists(json_file):
            # pyotb write
            pyotb_output.write(cache_file, pixel_type=pixel_type)
            # json
            with open(json_file, 'w', encoding='utf-8') as file:
                json.dump(summary, file, ensure_ascii=False, indent=4)

        super().__init__(filepath=cache_file.split("?&")[0])

__init__(pyotb_output, temporary_directory=None, output_parameter_key=None, extension=None, pixel_type=None, summary_modifier=None)

Initialize the cache.

Parameters:

Name Type Description Default
pyotb_output Output

a pyotb.Output instance.

 required
temporary_directory str

a temporary directory for the cached files. Default is system temp directory.

 None
output_parameter_key str

output parameter key (default is first key)

 None
extension str

file extension (default: .tif)

 None
pixel_type str

pixel type

 None
summary_modifier

optional function to modify the summary

 None
Source code in scenes/cache.py 
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
def __init__(
        self,
        pyotb_output: pyotb.Output,
        temporary_directory: str = None,
        output_parameter_key: str = None,
        extension: str = None,
        pixel_type: str = None,
        summary_modifier=None
):
    """
    Initialize the cache.

    Args:
        pyotb_output: a pyotb.Output instance.
        temporary_directory: a temporary directory for the cached files.
            Default is system temp directory.
        output_parameter_key: output parameter key (default is first key)
        extension: file extension (default: .tif)
        pixel_type: pixel type
        summary_modifier: optional function to modify the summary
    """
    # Get app
    pyotb_app = pyotb_output.parent_pyotb_app

    # Get summary
    assert isinstance(pyotb_app, pyotb.core.App)
    summary = pyotb.summarize(pyotb_app)  # need pyotb >= 1.5.1

    # Modify summary
    if summary_modifier:
        summary = summary_modifier(summary)

    # Summary --> md5sum
    desc = json.dumps(summary)
    md5sum = hashlib.md5(desc.encode('utf-8')).hexdigest()

    # App name
    app_name = summary["name"]

    # Cache filename
    if not output_parameter_key:
        output_parameter_key = pyotb_app.output_image_key
    if not extension:
        extension = ".tif?&gdal:co:COMPRESS=DEFLATE&gdal:co:BIGTIFF=YES"
    elif not extension.startswith("."):
        extension = f".{extension}"
    if not pixel_type:
        pixel_type = "float"
    if not temporary_directory:
        temporary_directory = tempfile.gettempdir()
    if not os.path.exists(temporary_directory):
        os.makedirs(temporary_directory, exist_ok=True)
    prefix = os.path.join(
        temporary_directory, f"{app_name}_{output_parameter_key}_{md5sum}"
    )
    cache_file = f"{prefix}{extension}"
    json_file = f"{prefix}.json"

    # Check which cache files already exist
    if not os.path.exists(json_file):
        # pyotb write
        pyotb_output.write(cache_file, pixel_type=pixel_type)
        # json
        with open(json_file, 'w', encoding='utf-8') as file:
            json.dump(summary, file, ensure_ascii=False, indent=4)

    super().__init__(filepath=cache_file.split("?&")[0])