    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.IOException;
    016import java.io.Serializable;
    017import java.lang.annotation.Annotation;
    018import java.lang.reflect.Field;
    019import java.util.ArrayList;
    020import java.util.Arrays;
    021import java.util.HashMap;
    022import java.util.List;
    023import java.util.Map;
    024
    025import org.eclipse.january.DatasetException;
    026import org.eclipse.january.IMonitor;
    027import org.eclipse.january.io.ILazyLoader;
    028import org.eclipse.january.metadata.MetadataFactory;
    029import org.eclipse.january.metadata.MetadataType;
    030import org.eclipse.january.metadata.OriginMetadata;
    031import org.eclipse.january.metadata.Reshapeable;
    032import org.eclipse.january.metadata.Sliceable;
    033import org.eclipse.january.metadata.Transposable;
    034
    035public class LazyDataset extends LazyDatasetBase implements Serializable, Cloneable {
    036        private static final long serialVersionUID = 2467865859867440242L;
    037
    038        protected int[]     oShape; // original shape
    039        protected long      size;   // number of items
    040        protected int       dtype;  // dataset type
    041        protected int       isize;  // number of elements per item
    042
    043        protected ILazyLoader loader;
    044        protected LazyDataset base = null; // used for transpose
    045
    046        // relative to loader or base
    047        protected int         prepShape = 0; // prepending and post-pending
    048        protected int         postShape = 0; // changes to shape
    049        protected int[]       begSlice = null; // slice begin
    050        protected int[]       delSlice = null; // slice delta
    051        protected int[]       map; // transposition map (same length as current shape)
    052        protected Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata = null;
    053
    054        /**
    055         * Create a lazy dataset
    056         * @param name
    057         * @param dtype dataset type
    058         * @param elements
    059         * @param shape
    060         * @param loader
    061         */
    062        public LazyDataset(String name, int dtype, int elements, int[] shape, ILazyLoader loader) {
    063                this.name = name;
    064                this.shape = shape.clone();
    065                this.oShape = this.shape;
    066                this.loader = loader;
    067                this.dtype = dtype;
    068                this.isize = elements;
    069                try {
    070                        size = ShapeUtils.calcLongSize(shape);
    071                } catch (IllegalArgumentException e) {
    072                        size = Long.MAX_VALUE; // this indicates that the entire dataset cannot be read in!
    073                }
    074        }
    075
    076        /**
    077         * Create a lazy dataset
    078         * @param name
    079         * @param dtype dataset type
    080         * @param shape
    081         * @param loader
    082         */
    083        public LazyDataset(String name, int dtype, int[] shape, ILazyLoader loader) {
    084                this(name, dtype, 1, shape, loader);
    085        }
    086
    087        /**
    088         * Create a lazy dataset based on in-memory data (handy for testing)
    089         * @param dataset
    090         */
    091        public static LazyDataset createLazyDataset(final Dataset dataset) {
    092                return new LazyDataset(dataset.getName(), dataset.getDType(), dataset.getElementsPerItem(), dataset.getShape(),
    093                new ILazyLoader() {
    094                        private static final long serialVersionUID = -6725268922780517523L;
    095
    096                        final Dataset d = dataset;
    097                        @Override
    098                        public boolean isFileReadable() {
    099                                return true;
    100                        }
    101
    102                        @Override
    103                        public Dataset getDataset(IMonitor mon, SliceND slice) throws IOException {
    104                                return d.getSlice(mon, slice);
    105                        }
    106                });
    107        }
    108
    109        /**
    110         * Can return -1 for unknown
    111         */
    112        @Override
    113        public int getDType() {
    114                return dtype;
    115        }
    116
    117        /**
    118         * Can return -1 for unknown
    119         */
    120        @Override
    121        public int getElementsPerItem() {
    122                return isize;
    123        }
    124
    125        @Override
    126        public int getSize() {
    127                return (int) size;
    128        }
    129
    130        @Override
    131        public String toString() {
    132                StringBuilder out = new StringBuilder();
    133
    134                if (name != null && name.length() > 0) {
    135                        out.append("Lazy dataset '");
    136                        out.append(name);
    137                        out.append("' has shape [");
    138                } else {
    139                        out.append("Lazy dataset shape is [");
    140                }
    141                int rank = shape == null ? 0 : shape.length;
    142
    143                if (rank > 0 && shape[0] >= 0) {
    144                        out.append(shape[0]);
    145                }
    146                for (int i = 1; i < rank; i++) {
    147                        out.append(", " + shape[i]);
    148                }
    149                out.append(']');
    150
    151                return out.toString();
    152        }
    153
    154        @Override
    155        public boolean equals(Object obj) {
    156                if (!super.equals(obj))
    157                        return false;
    158
    159                LazyDataset other = (LazyDataset) obj;
    160                if (dtype != other.dtype) {
    161                        return false;
    162                }
    163                if (isize != other.isize) {
    164                        return false;
    165                }
    166
    167                if (!Arrays.equals(shape, other.shape)) {
    168                        return false;
    169                }
    170
    171                if (loader != other.loader) {
    172                        return false;
    173                }
    174
    175                if (prepShape != other.prepShape) {
    176                        return false;
    177                }
    178
    179                if (postShape != other.postShape) {
    180                        return false;
    181                }
    182
    183                if (!Arrays.equals(begSlice, other.begSlice)) {
    184                        return false;
    185                }
    186                if (!Arrays.equals(delSlice, other.delSlice)) {
    187                        return false;
    188                }
    189                if (!Arrays.equals(map, other.map)) {
    190                        return false;
    191                }
    192                return true;
    193        }
    194
    195        @Override
    196        public LazyDataset clone() {
    197                LazyDataset ret = new LazyDataset(new String(name), dtype, isize, oShape, loader);
    198                ret.shape = shape;
    199                ret.size = size;
    200                ret.prepShape = prepShape;
    201                ret.postShape = postShape;
    202                ret.begSlice = begSlice;
    203                ret.delSlice = delSlice;
    204                ret.map = map;
    205                ret.base = base;
    206                ret.metadata = copyMetadata();
    207                ret.oMetadata = oMetadata;
    208                return ret;
    209        }
    210
    211        @Override
    212        public void setShape(int... shape) {
    213                setShapeInternal(shape);
    214        }
    215
    216        @Override
    217        public LazyDataset squeezeEnds() {
    218                setShapeInternal(ShapeUtils.squeezeShape(shape, true));
    219                return this;
    220        }
    221
    222        @Override
    223        public Dataset getSlice(int[] start, int[] stop, int[] step) throws DatasetException {
    224                return getSlice(null, start, stop, step);
    225        }
    226
    227        @Override
    228        public Dataset getSlice(Slice... slice) throws DatasetException {
    229                if (slice == null || slice.length == 0) {
    230                        return getSlice(null, new SliceND(shape));
    231                }
    232                return getSlice(null, new SliceND(shape, slice));
    233        }
    234
    235        @Override
    236        public Dataset getSlice(SliceND slice) throws DatasetException {
    237                return getSlice(null, slice);
    238        }
    239
    240        @Override
    241        public Dataset getSlice(IMonitor monitor, Slice... slice) throws DatasetException {
    242                if (slice == null || slice.length == 0) {
    243                        return getSlice(monitor, new SliceND(shape));
    244                }
    245                return getSlice(monitor, new SliceND(shape, slice));
    246        }
    247
    248        @Override
    249        public LazyDataset getSliceView(Slice... slice) {
    250                if (slice == null || slice.length == 0) {
    251                        return getSliceView(new SliceND(shape));
    252                }
    253                return getSliceView(new SliceND(shape, slice));
    254        }
    255
    256        /**
    257         * @param nShape
    258         */
    259        private void setShapeInternal(int... nShape) {
    260
    261                long nsize = ShapeUtils.calcLongSize(nShape);
    262                if (nsize != size) {
    263                        throw new IllegalArgumentException("Size of new shape is not equal to current size");
    264                }
    265
    266                if (nsize == 1) {
    267                        shape = nShape.clone();
    268                } else {
    269                        int ob = -1; // first non-unit dimension
    270                        int or = shape.length;
    271                        for (int i = 0; i < or; i++) {
    272                                if (shape[i] != 1) {
    273                                        ob = i;
    274                                        break;
    275                                }
    276                        }
    277                        assert ob >= 0;
    278                        int oe = -1; // last non-unit dimension
    279                        for (int i = or - 1; i >= ob; i--) {
    280                                if (shape[i] != 1) {
    281                                        oe = i;
    282                                        break;
    283                                }
    284                        }
    285                        assert oe >= 0;
    286                        oe++;
    287
    288                        int nb = -1; // first non-unit dimension
    289                        int nr = nShape.length;
    290                        for (int i = 0; i < nr; i++) {
    291                                if (nShape[i] != 1) {
    292                                        nb = i;
    293                                        break;
    294                                }
    295                        }
    296
    297                        int i = ob;
    298                        int j = nb;
    299                        if (begSlice == null) {
    300                                for (; i < oe && j < nr; i++, j++) {
    301                                        if (shape[i] != nShape[j]) {
    302                                                throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape");
    303                                        }
    304                                }
    305                        } else {
    306                                int[] nBegSlice = new int[nr];
    307                                int[] nDelSlice = new int[nr];
    308                                Arrays.fill(nDelSlice, 1);
    309                                for (; i < oe && j < nr; i++, j++) {
    310                                        if (shape[i] != nShape[j]) {
    311                                                throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape");
    312                                        }
    313                                        nBegSlice[j] = begSlice[i];
    314                                        nDelSlice[j] = delSlice[i];
    315                                }
    316
    317                                begSlice = nBegSlice;
    318                                delSlice = nDelSlice;
    319                        }
    320                        prepShape += nb - ob;
    321                        postShape += nr - oe;
    322                }
    323
    324                if (metadata != null) {
    325                        storeMetadata(metadata, Reshapeable.class);
    326                        metadata = copyMetadata();
    327                        reshapeMetadata(shape, nShape);
    328                }
    329                shape = nShape;
    330        }
    331
    332        @Override
    333        public LazyDataset getSliceView(int[] start, int[] stop, int[] step) {
    334                return getSliceView(new SliceND(shape, start, stop, step));
    335        }
    336
    337        @Override
    338        public LazyDataset getSliceView(SliceND slice) {
    339                LazyDataset view = clone();
    340                if (slice.isAll()) {
    341                        return view;
    342                }
    343
    344                int[] lstart = slice.getStart();
    345                int[] lstep  = slice.getStep();
    346                final int rank = shape.length;
    347
    348                int[] nShape = slice.getShape();
    349                view.shape = nShape;
    350                view.size = ShapeUtils.calcLongSize(nShape);
    351                if (begSlice == null) {
    352                        view.begSlice = lstart.clone();
    353                        view.delSlice = lstep.clone();
    354                } else {
    355                        view.begSlice = new int[rank];
    356                        view.delSlice = new int[rank];
    357                        for (int i = 0; i < rank; i++) {
    358                                view.begSlice[i] = begSlice[i] + lstart[i] * delSlice[i];
    359                                view.delSlice[i] = delSlice[i] * lstep[i];
    360                        }
    361                }
    362                view.storeMetadata(metadata, Sliceable.class);
    363
    364                view.sliceMetadata(true, slice);
    365                return view;
    366        }
    367
    368        @Override
    369        public LazyDataset getTransposedView(int... axes) {
    370                LazyDataset view = clone();
    371
    372                // everything now is seen through a map
    373                axes = checkPermutatedAxes(shape, axes);
    374                if (axes == null) {
    375                        return view;
    376                }
    377
    378                int r = shape.length;
    379                view.shape = new int[r];
    380                for (int i = 0; i < r; i++) {
    381                        view.shape[i] = shape[axes[i]];
    382                }
    383
    384                view.prepShape = 0;
    385                view.postShape = 0;
    386                view.begSlice = null;
    387                view.delSlice = null;
    388                view.map = axes;
    389                view.base = this;
    390                view.storeMetadata(metadata, Transposable.class);
    391                view.transposeMetadata(axes);
    392                return view;
    393        }
    394
    395        @Override
    396        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws DatasetException {
    397                return getSlice(monitor, new SliceND(shape, start, stop, step));
    398        }
    399
    400        @Override
    401        public Dataset getSlice(IMonitor monitor, SliceND slice) throws DatasetException {
    402
    403                if (loader != null && !loader.isFileReadable()) {
    404                        return null; // TODO add interaction to use plot (or remote) server to load dataset
    405                }
    406
    407                SliceND nslice = calcTrueSlice(slice);
    408
    409                Dataset a;
    410                if (base != null) {
    411                        a = base.getSlice(monitor, nslice);
    412                } else {
    413                        try {
    414                                a = DatasetUtils.convertToDataset(loader.getDataset(monitor, nslice));
    415                        } catch (IOException e) {
    416                                logger.error("Problem getting {}: {}", String.format("slice %s %s %s from %s", Arrays.toString(slice.getStart()), Arrays.toString(slice.getStop()),
    417                                                                Arrays.toString(slice.getStep()), loader), e);
    418                                throw new DatasetException(e);
    419                        }
    420                        a.setName(name + AbstractDataset.BLOCK_OPEN + nslice.toString() + AbstractDataset.BLOCK_CLOSE);
    421                        if (metadata != null && a instanceof LazyDatasetBase) {
    422                                LazyDatasetBase ba = (LazyDatasetBase) a;
    423                                ba.metadata = copyMetadata();
    424                                if (oMetadata != null) {
    425                                        ba.restoreMetadata(oMetadata);
    426                                }
    427                                //metadata axis may be larger than data
    428                                if (!nslice.isAll() || nslice.getMaxShape() != nslice.getShape()) {
    429                                        ba.sliceMetadata(true, nslice);
    430                                }
    431                        }
    432                }
    433                if (map != null) {
    434                        a = a.getTransposedView(map);
    435                }
    436                if (slice != null) {
    437                        a.setShape(slice.getShape());
    438                }
    439                a.addMetadata(MetadataFactory.createMetadata(OriginMetadata.class, this, nslice.convertToSlice(), oShape, null, name));
    440
    441                return a;
    442        }
    443
    444        // reverse transform
    445        private int[] getOriginal(int[] values) {
    446                if (values == null) {
    447                        return null;
    448                }
    449                int r = values.length;
    450                if (map == null || r < 2) {
    451                        return values;
    452                }
    453                int[] ovalues = new int[r];
    454                for (int i = 0; i < r; i++) {
    455                        ovalues[map[i]] = values[i];
    456                }
    457                return ovalues;
    458        }
    459
    460        protected final SliceND calcTrueSlice(SliceND slice) {
    461                if (slice == null) {
    462                        slice = new SliceND(shape);
    463                }
    464                int[] lstart = slice.getStart();
    465                int[] lstop  = slice.getStop();
    466                int[] lstep  = slice.getStep();
    467
    468                int[] nstart;
    469                int[] nstop;
    470                int[] nstep;
    471
    472                int r = base == null ? oShape.length : base.shape.length;
    473                nstart = new int[r];
    474                nstop = new int[r];
    475                nstep = new int[r];
    476                Arrays.fill(nstop, 1);
    477                Arrays.fill(nstep, 1);
    478                {
    479                        int i = 0;
    480                        int j = 0;
    481                        if (prepShape < 0) { // ignore entries from new slice
    482                                i = -prepShape;
    483                        } else if (prepShape > 0) {
    484                                j = prepShape;
    485                        }
    486                        if (begSlice == null) {
    487                                for (; i < r && j < shape.length; i++, j++) {
    488                                        nstart[i] = lstart[j];
    489                                        nstop[i]  = lstop[j];
    490                                        int d = lstep[j];
    491                                        if (d < 0 && nstop[i] < 0) { // need to wrap around further
    492                                                int l = base == null ? oShape[j]: base.shape[j];
    493                                                nstop[i] -= l;
    494                                        }
    495                                        nstep[i]  = d;
    496                                }
    497                        } else {
    498                                for (; i < r && j < shape.length; i++, j++) {
    499                                        int b = begSlice[j];
    500                                        int d = delSlice[j];
    501                                        nstart[i] = b + lstart[j] * d;
    502                                        nstop[i]  = b + (lstop[j] - 1) * d + (d >= 0 ? 1 : -1);
    503                                        if (d < 0 && nstop[i] < 0) { // need to wrap around further
    504                                                int l = base == null ? oShape[j]: base.shape[j];
    505                                                nstop[i] -= l;
    506                                        }
    507                                        nstep[i]  = lstep[j] * d;
    508                                }
    509                        }
    510                        if (map != null) {
    511                                nstart = getOriginal(nstart);
    512                                nstop  = getOriginal(nstop);
    513                                nstep  = getOriginal(nstep);
    514                        }
    515                }
    516
    517                return createSlice(nstart, nstop, nstep);
    518        }
    519
    520        protected final IDataset transformInput(IDataset data) {
    521                if (map == null) {
    522                        return data;
    523                }
    524                return data.getTransposedView(map);
    525        }
    526
    527        protected SliceND createSlice(int[] nstart, int[] nstop, int[] nstep) {
    528                return new SliceND(base == null ? oShape : base.shape, nstart, nstop, nstep);
    529        }
    530
    531        /**
    532         * Store metadata items that has given annotation
    533         * @param origMetadata
    534         * @param aclazz
    535         */
    536        private void storeMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> origMetadata, Class<? extends Annotation> aclazz) {
    537                List<Class<? extends MetadataType>> mclazzes = findAnnotatedMetadata(aclazz);
    538                if (mclazzes.size() == 0) {
    539                        return;
    540                }
    541
    542                if (oMetadata == null) {
    543                        oMetadata = new HashMap<Class<? extends MetadataType>, List<MetadataType>>();
    544                }
    545                for (Class<? extends MetadataType> mc : mclazzes) {
    546                        if (oMetadata.containsKey(mc)) {
    547                                continue; // do not overwrite original
    548                        }
    549
    550                        List<MetadataType> l = origMetadata.get(mc);
    551                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
    552                        for (MetadataType m : l) {
    553                                nl.add(m.clone());
    554                        }
    555                        oMetadata.put(mc, nl);
    556                }
    557        }
    558
    559        @SuppressWarnings("unchecked")
    560        private List<Class<? extends MetadataType>> findAnnotatedMetadata(Class<? extends Annotation> aclazz) {
    561                List<Class<? extends MetadataType>> mclazzes = new ArrayList<Class<? extends MetadataType>>();
    562                if (metadata == null) {
    563                        return mclazzes;
    564                }
    565
    566                for (Class<? extends MetadataType> c : metadata.keySet()) {
    567                        boolean hasAnn = false;
    568                        for (MetadataType m : metadata.get(c)) {
    569                                if (m == null) {
    570                                        continue;
    571                                }
    572
    573                                Class<? extends MetadataType> mc = m.getClass();
    574                                do { // iterate over super-classes
    575                                        for (Field f : mc.getDeclaredFields()) {
    576                                                if (f.isAnnotationPresent(aclazz)) {
    577                                                        hasAnn = true;
    578                                                        break;
    579                                                }
    580                                        }
    581                                        Class<?> sclazz = mc.getSuperclass();
    582                                        if (!MetadataType.class.isAssignableFrom(sclazz)) {
    583                                                break;
    584                                        }
    585                                        mc = (Class<? extends MetadataType>) sclazz;
    586                                } while (!hasAnn);
    587                                if (hasAnn) {
    588                                        break;
    589                                }
    590                        }
    591                        if (hasAnn) {
    592                                mclazzes.add(c);
    593                        }
    594                }
    595                return mclazzes;
    596        }
    597
    598        /**
    599         * Gets the maximum size of a slice of a dataset in a given dimension
    600         * which should normally fit in memory. Note that it might be possible
    601         * to get more in memory, this is a conservative estimate and seems to
    602         * almost always work at the size returned; providing Xmx is less than
    603         * the physical memory.
    604         *
    605         * To get more in memory increase -Xmx setting or use an expression
    606         * which calls a rolling function (like rmean) instead of slicing directly
    607         * to memory.
    608         *
    609         * @param lazySet
    610         * @param dimension
    611         * @return maximum size of dimension that can be sliced.
    612         */
    613        public static int getMaxSliceLength(ILazyDataset lazySet, int dimension) {
    614                // size in bytes of each item
    615                final double size = DTypeUtils.getItemBytes(DTypeUtils.getDTypeFromClass(lazySet.getElementClass()), lazySet.getElementsPerItem());
    616
    617                // Max in bytes takes into account our minimum requirement
    618                final double max  = Math.max(Runtime.getRuntime().totalMemory(), Runtime.getRuntime().maxMemory());
    619
    620                // Firstly if the whole dataset it likely to fit in memory, then we allow it.
    621                // Space specified in bytes per item available
    622                final double space = max/lazySet.getSize();
    623
    624                // If we have room for this whole dataset, then fine
    625                int[] shape = lazySet.getShape();
    626                if (space >= size) {
    627                        return shape[dimension];
    628                }
    629
    630                // Otherwise estimate what we can fit in, conservatively.
    631                // First get size of one slice, see it that fits, if not, still return 1
    632                double sizeOneSlice = size; // in bytes
    633                for (int dim = 0; dim < shape.length; dim++) {
    634                        if (dim == dimension) {
    635                                continue;
    636                        }
    637                        sizeOneSlice *= shape[dim];
    638                }
    639                double avail = max / sizeOneSlice;
    640                if (avail < 1) {
    641                        return 1;
    642                }
    643
    644                // We fudge this to leave some room
    645                return (int) Math.floor(avail/4d);
    646        }
    647}