hrm_omero.transfer

  1"""Transfer related functions."""
  2
  3import os
  4import tempfile
  5from io import BytesIO
  6from pathlib import Path
  7import stat
  8
  9from loguru import logger as log
 10from PIL import Image
 11
 12from . import hrm
 13from .decorators import connect_and_set_group
 14from .omero import extract_image_id, add_annotation_keyvalue
 15from .misc import printlog, changemodes
 16
 17
 18@connect_and_set_group
 19def from_omero(conn, omero_id, dest):
 20    """Download the corresponding original file(s) from an image ID.
 21
 22    This only works for image IDs that were created with OMERO 5.0 or later as previous
 23    versions don't have an "original file" linked to an image.
 24
 25    Note that files will be downloaded with their original name, which is not
 26    necessarily the name shown by OMERO, e.g. if an image name was changed in OMERO. To
 27    keep filesets consistent, we have to preserve the original names!
 28
 29    In addition to the original file(s), it also downloads a thumbnail of the requested
 30    file from OMERO and puts it into the appropriate place so HRM will show it as a
 31    preview until the user hits "re-generate preview".
 32
 33    Parameters
 34    ----------
 35    conn : omero.gateway.BlitzGateway
 36        The OMERO connection object.
 37    omero_id : hrm_omero.misc.OmeroId
 38        The ID of the OMERO image to be downloaded.
 39    dest : str
 40        The destination path.
 41
 42    Returns
 43    -------
 44    bool
 45        True in case the download was successful, False otherwise.
 46
 47    Raises
 48    ------
 49    ValueError
 50        Raised in case an object that is not of type `Image` was requested.
 51    """
 52    log.trace(f"Downloading {omero_id.obj_type}:{omero_id.obj_id} to [{dest}]...")
 53
 54    # conn.setGroupForSession(-1)
 55    conn.SERVICE_OPTS.setOmeroGroup(-1)  # still working with OMERO-5.6.3
 56    if omero_id.obj_type != "Image":
 57        raise ValueError("Currently only the download of 'Image' objects is supported!")
 58
 59    # check if dest is a directory, rewrite it otherwise:
 60    if not os.path.isdir(dest):
 61        # FIXME: this should raise a ValueError as it's quite counter-intuitive that
 62        # specifying a file *name* for the target doesn't have an effect on how the
 63        # downloaded file will be called actually!
 64        dest = os.path.dirname(dest)
 65    from omero_model_OriginalFileI import OriginalFileI
 66
 67    # use image objects and getFileset() methods to determine original files,
 68    # see the following OME forum thread for some more details:
 69    # https://www.openmicroscopy.org/community/viewtopic.php?f=6&t=7563
 70    target_obj = conn.getObject(omero_id.obj_type, omero_id.obj_id)
 71    if not target_obj:
 72        printlog("ERROR", f"ERROR: can't find image with ID [{omero_id.obj_id}]!")
 73        return False
 74
 75    fset = target_obj.getFileset()
 76    if not fset:  # pragma: no cover
 77        printlog("ERROR", f"ERROR: no original file(s) for [{omero_id.obj_id}] found!")
 78        return False
 79
 80    # NOTE: the idea of offering to download the OME-TIFF from OMERO (i.e. the converted
 81    # data) as an alternative has been discarded for the moment - see upstream HRM
 82    # ticket #398 (http://hrm.svi.nl:8080/redmine/issues/398)
 83    downloads = []
 84    # assemble a list of items to download
 85    for fset_file in fset.listFiles():
 86        # for foo in dir(fset_file):
 87        #     log.error(foo)
 88        file_name = fset_file.getName()
 89        file_path = fset_file.getPath()
 90        downloads.append((fset_file.getId(), os.path.join(file_path, file_name)))
 91
 92    # determine the common prefix, e.g. `hrm-test-02_3/2022-02/17/14-34-16.480/`
 93    all_files = [x[1] for x in downloads]
 94    strip = os.path.commonprefix(all_files).rindex("/") + 1
 95
 96    # strip the common prefix, check if any of the files already exist:
 97    top_level = []
 98    for i, pair in enumerate(downloads):
 99        rel_path = pair[1][strip:]
100        log.trace(f"relative path: {rel_path}")
101        top_level.append(rel_path.split("/")[0])
102        abs_path = os.path.join(dest, rel_path)
103        downloads[i] = (pair[0], abs_path)
104        if os.path.exists(abs_path):
105            printlog("ERROR", f"ERROR: file '{abs_path}' already existing!")
106            return False
107
108    # now initiate the downloads for all original files:
109    for (file_id, tgt) in downloads:
110        try:
111            log.trace(f"Downloading original file [{file_id}] to [{tgt}]...")
112            os.makedirs(os.path.dirname(tgt), exist_ok=True)
113            conn.c.download(OriginalFileI(file_id), tgt)
114        except Exception as err:  # pylint: disable-msg=broad-except
115            printlog("ERROR", f"ERROR: downloading {file_id} to '{tgt}' failed: {err}")
116            return False
117        printlog("SUCCESS", f"ID {file_id} downloaded as '{os.path.basename(tgt)}'")
118
119    changemodes(dest, top_level)
120
121    # NOTE: for filesets with a single file or e.g. ICS/IDS pairs it makes
122    # sense to use the target name of the first file to construct the name for
123    # the thumbnail, but it is unclear whether this is a universal approach:
124    fetch_thumbnail(conn, omero_id, downloads[0][1])
125    return True
126
127
128def fetch_thumbnail(conn, omero_id, dest):
129    """Download the thumbnail of a given image from OMERO.
130
131    OMERO provides thumbnails for stored images, this function downloads the thumbnail
132    image and places it as preview in the corresponding HRM directory.
133
134    Parameters
135    ----------
136    conn : omero.gateway.BlitzGateway
137        The OMERO connection object.
138    omero_id : hrm_omero.misc.OmeroId
139        The ID of the OMERO image to fetch the thumbnail for.
140    dest : str
141        The destination filename.
142
143    Returns
144    -------
145    bool
146        True in case the download was successful, False otherwise.
147    """
148    ##### WARNING - WARNING - WARNING - WARNING - WARNING - WARNING - WARNING #####
149    #
150    # do NOT decorate this function using `@connect_and_set_group`, see the
151    # explanation in the decorator definition for more details
152    #
153    ##### WARNING - WARNING - WARNING - WARNING - WARNING - WARNING - WARNING #####
154    log.info(f"Trying to fetch thumbnail for OMERO image [{omero_id.obj_id}]...")
155    log.trace(f"Requested target location: [{dest}].")
156
157    base_dir, fname = os.path.split(dest)
158    target_dir = Path(base_dir) / "hrm_previews"
159    target = Path(target_dir) / f"{fname}.preview_xy.jpg"
160
161    image_obj = conn.getObject("Image", omero_id.obj_id)
162    image_data = image_obj.getThumbnail()
163    log.trace(f"len(image_data)={len(image_data)}")
164    thumbnail = Image.open(BytesIO(image_data))
165    try:
166        target_dir.mkdir(parents=True, exist_ok=True)
167        # TODO: consider using `misc.changemodes()`
168        # for an unkown reason using mode=(S_IRWXU | S_IRWXG) on the `mkdir()` call
169        # doesn't seem to work, so we have to add group-write in a second step:
170        target_dir.chmod(target_dir.stat().st_mode | stat.S_IWGRP)
171        thumbnail.save(target.as_posix(), format="jpeg")
172        printlog("SUCCESS", f"Thumbnail downloaded to '{target}'.")
173        target.chmod(target.stat().st_mode | stat.S_IWGRP)
174        log.success(f"Added group-write permissions to '{target}'.")
175    except Exception as err:  # pylint: disable-msg=broad-except
176        printlog("ERROR", f"ERROR downloading thumbnail to '{target}': {err}")
177        return False
178
179    return True
180
181
182@connect_and_set_group
183def to_omero(conn, omero_id, image_file, omero_logfile="", _fetch_zip_only=False):
184    """Upload an image into a specific dataset in OMERO.
185
186    In case we know from the suffix that a given  format is not supported by OMERO, the
187    upload will not be initiated at all (e.g. for SVI-HDF5, having the suffix '.h5').
188
189    The import itself is done by instantiating the CLI class, assembling the required
190    arguments, and finally running `cli.invoke()`. This eventually triggers the
191    `importer()` method defined in [OMERO's Python bindings][1].
192
193    [1]: https://github.com/ome/omero-py/blob/master/src/omero/plugins/import.py
194
195    Parameters
196    ----------
197    conn : omero.gateway.BlitzGateway
198        The OMERO connection object.
199    omero_id : hrm_omero.misc.OmeroId
200        The ID of the target dataset in OMERO.
201    image_file : str
202        The local image file including the full path.
203    omero_logfile : str, optional
204        The prefix of files to be used to capture OMERO's `import` call stderr messages.
205        If the parameter is non-empty the `--debug ALL` option will be added to the
206        `omero` call with the output being placed in the specified file. If the
207        parameter is omitted or empty, debug messages will be disabled.
208    _fetch_zip_only : bool, optional
209        Replaces all parameters to the import call by `--advanced-help`, which is
210        **intended for INTERNAL TESTING ONLY**. No actual import will be attempted!
211
212    Returns
213    -------
214    bool
215        True in case of success, False otherwise.
216
217    Raises
218    ------
219    TypeError
220        Raised in case `image_file` is in a format that is not supported by OMERO.
221    ValueError
222        Raised in case `omero_id` is not pointing to a dataset.
223    """
224
225    # TODO: revisit this, as e.g. BDV .h5 files are supported for now!
226    if image_file.lower().endswith((".h5", ".hdf5")):
227        msg = f"ERROR importing [{image_file}]: HDF5 format not supported by OMERO!"
228        printlog("ERROR", msg)
229        raise TypeError(msg)
230
231    if omero_id.obj_type != "Dataset":
232        msg = "Currently only the upload to 'Dataset' objects is supported!"
233        printlog("ERROR", msg)
234        raise ValueError(msg)
235
236    # we have to create the annotations *before* we actually upload the image
237    # data itself and link them to the image during the upload - the other way
238    # round is not possible right now as the CLI wrapper (see below) doesn't
239    # expose the ID of the newly created object in OMERO (confirmed by J-M and
240    # Sebastien on the 2015 OME Meeting):
241    #### namespace = "deconvolved.hrm"
242    #### mime = 'text/plain'
243    #### annotations = []
244    #### # TODO: the list of suffixes should not be hardcoded here!
245    #### for suffix in ['.hgsb', '.log.txt', '.parameters.txt']:
246    ####     if not os.path.exists(basename + suffix):
247    ####         continue
248    ####     ann = conn.createFileAnnfromLocalFile(
249    ####         basename + suffix, mimetype=mime, ns=namespace, desc=None)
250    ####     annotations.append(ann.getId())
251
252    # currently there is no direct "Python way" to import data into OMERO, so we have to
253    # use the CLI wrapper for this...
254    # TODO: check the more recent code mentioned by the OME developers in the forum
255    # thread: https://forum.image.sc/t/automated-uploader-to-omero-in-python/38290
256    # https://gitlab.com/openmicroscopy/incubator/omero-python-importer/-/blob/master/import.py)
257    # and also see https://pypi.org/project/omero-upload/
258    from omero.cli import CLI
259
260    cli = CLI()
261    cli.loadplugins()
262    cli.set_client(conn.c)
263    import_args = ["import"]
264
265    # disable upgrade checks (https://forum.image.sc/t/unable-to-use-cli-importer/26424)
266    import_args.extend(["--skip", "upgrade"])
267
268    if omero_logfile:
269        log.warning(f"Messages (stderr) from import will go to [{omero_logfile}].")
270        import_args.extend(["--debug", "ALL"])
271        import_args.extend(["--errs", omero_logfile])
272
273    import_args.extend(["-d", omero_id.obj_id])
274
275    # capture stdout and request YAML format to parse the output later on:
276    tempdir = tempfile.TemporaryDirectory(prefix="hrm-omero__")
277    cap_stdout = f"{tempdir.name}/omero-import-stdout"
278    log.debug(f"Capturing stdout of the 'omero' call into [{cap_stdout}]...")
279    import_args.extend(["--file", cap_stdout])
280    import_args.extend(["--output", "yaml"])
281
282    #### for ann_id in annotations:
283    ####     import_args.extend(['--annotation_link', str(ann_id)])
284    import_args.append(image_file)
285    if _fetch_zip_only:
286        # calling 'import --advanced-help' will trigger the download of OMERO.java.zip
287        # in case it is not yet present (the extract_image_id() call will then fail,
288        # resulting in the whole function returning "False")
289        printlog("WARNING", "As '_fetch_zip_only' is set NO IMPORT WILL BE ATTEMPTED!")
290        import_args = ["import", "--advanced-help"]
291    log.debug(f"import_args: {import_args}")
292    try:
293        cli.invoke(import_args, strict=True)
294        imported_id = extract_image_id(cap_stdout)
295        log.success(f"Imported OMERO image ID: {imported_id}")
296    except PermissionError as err:
297        printlog("ERROR", err)
298        omero_userdir = os.environ.get("OMERO_USERDIR", "<not-set>")
299        printlog("ERROR", f"Current OMERO_USERDIR value: {omero_userdir}")
300        printlog(
301            "ERROR",
302            (
303                "Please make sure to read the documentation about the 'OMERO_USERDIR' "
304                "environment variable and also check if the file to be imported has "
305                "appropriate permissions!"
306            ),
307        )
308        return False
309    except Exception as err:  # pylint: disable-msg=broad-except
310        printlog("ERROR", f"ERROR: uploading '{image_file}' to {omero_id} failed!")
311        printlog("ERROR", f"OMERO error message: >>>{err}<<<")
312        printlog("WARNING", f"import_args: {import_args}")
313        return False
314    finally:
315        tempdir.cleanup()
316
317    target_id = f"G:{omero_id.group}:Image:{imported_id}"
318    try:
319        summary = hrm.parse_summary(image_file)
320        add_annotation_keyvalue(conn, target_id, summary)
321    except Exception as err:  # pragma: no cover # pylint: disable-msg=broad-except
322        log.error(f"Creating a parameter summary from [{image_file}] failed: {err}")
323
324    return True