FM / package

/usr/local/lib64/python3.6/site-packages/torch/package/

/usr/local/lib64/python3.6/site-packages/torch/package

Name	Size	Mode	Actions
analyze/	-	0755	rm
__pycache__/	-	0755	rm
file_structure_representation.py	4699	0644	edit dl rm
find_file_dependencies.py	4138	0644	edit dl rm
glob_group.py	3610	0644	edit dl rm
importer.py	7948	0644	edit dl rm
package_exporter.py	43710	0644	edit dl rm
package_importer.py	27362	0644	edit dl rm
_digraph.py	2633	0644	edit dl rm
_directory_reader.py	1827	0644	edit dl rm
_importlib.py	2989	0644	edit dl rm
_mangling.py	1854	0644	edit dl rm
_mock.py	2838	0644	edit dl rm
_package_pickler.py	3739	0644	edit dl rm
_package_unpickler.py	935	0644	edit dl rm
_stdlib.py	13452	0644	edit dl rm
__init__.py	388	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/torch/package/_directory_reader.py (1827B)

import os.path
from glob import glob
from typing import Any, List

import torch

_storages: List[Any] = [
    torch.DoubleStorage,
    torch.FloatStorage,
    torch.LongStorage,
    torch.IntStorage,
    torch.ShortStorage,
    torch.CharStorage,
    torch.ByteStorage,
    torch.BoolStorage,
]
_dtype_to_storage = {data_type(0).dtype: data_type for data_type in _storages}

# because get_storage_from_record returns a tensor!?
class _HasStorage(object):
    def __init__(self, storage):
        self._storage = storage

def storage(self):
        return self._storage

class DirectoryReader(object):
    """
    Class to allow PackageImporter to operate on unzipped packages. Methods
    copy the behavior of the internal PyTorchFileReader class (which is used for
    accessing packages in all other cases).

N.B.: ScriptObjects are not depickleable or accessible via this DirectoryReader
    class due to ScriptObjects requiring an actual PyTorchFileReader instance.
    """

def __init__(self, directory):
        self.directory = directory

def get_record(self, name):
        filename = f"{self.directory}/{name}"
        with open(filename, "rb") as f:
            return f.read()

def get_storage_from_record(self, name, numel, dtype):
        storage = _dtype_to_storage[dtype]
        filename = f"{self.directory}/{name}"
        return _HasStorage(storage.from_file(filename=filename, size=numel))

def has_record(self, path):
        full_path = os.path.join(self.directory, path)
        return os.path.isfile(full_path)

def get_all_records(
        self,
    ):
        files = []
        for filename in glob(f"{self.directory}/**", recursive=True):
            if not os.path.isdir(filename):
                files.append(filename[len(self.directory) + 1 :])
        return files

import os.path from glob import glob from typing import Any, List import torch _storages: List[Any] = [ torch.DoubleStorage, torch.FloatStorage, torch.LongStorage, torch.IntStorage, torch.ShortStorage, torch.CharStorage, torch.ByteStorage, torch.BoolStorage, ] _dtype_to_storage = {data_type(0).dtype: data_type for data_type in _storages} # because get_storage_from_record returns a tensor!? class _HasStorage(object): def __init__(self, storage): self._storage = storage def storage(self): return self._storage class DirectoryReader(object): """ Class to allow PackageImporter to operate on unzipped packages. Methods copy the behavior of the internal PyTorchFileReader class (which is used for accessing packages in all other cases). N.B.: ScriptObjects are not depickleable or accessible via this DirectoryReader class due to ScriptObjects requiring an actual PyTorchFileReader instance. """ def __init__(self, directory): self.directory = directory def get_record(self, name): filename = f"{self.directory}/{name}" with open(filename, "rb") as f: return f.read() def get_storage_from_record(self, name, numel, dtype): storage = _dtype_to_storage[dtype] filename = f"{self.directory}/{name}" return _HasStorage(storage.from_file(filename=filename, size=numel)) def has_record(self, path): full_path = os.path.join(self.directory, path) return os.path.isfile(full_path) def get_all_records( self, ): files = [] for filename in glob(f"{self.directory}/**", recursive=True): if not os.path.isdir(filename): files.append(filename[len(self.directory) + 1 :]) return files