    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.IOException;
    016import java.io.Serializable;
    017import java.lang.annotation.Annotation;
    018import java.lang.reflect.Field;
    019import java.util.ArrayList;
    020import java.util.Arrays;
    021import java.util.HashMap;
    022import java.util.LinkedList;
    023import java.util.List;
    024import java.util.Map;
    025
    026import org.eclipse.january.DatasetException;
    027import org.eclipse.january.IMonitor;
    028import org.eclipse.january.io.ILazyLoader;
    029import org.eclipse.january.metadata.MetadataFactory;
    030import org.eclipse.january.metadata.MetadataType;
    031import org.eclipse.january.metadata.OriginMetadata;
    032import org.eclipse.january.metadata.Reshapeable;
    033import org.eclipse.january.metadata.Sliceable;
    034import org.eclipse.january.metadata.Transposable;
    035
    036public class LazyDataset extends LazyDatasetBase implements Serializable, Cloneable {
    037        private static final long serialVersionUID = 2467865859867440242L;
    038
    039        protected Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata = null;
    040        protected int[] oShape; // original shape
    041        protected long  size;   // number of items
    042        private Class<? extends Dataset> clazz = null;
    043        protected int   isize;  // number of elements per item
    044
    045        protected ILazyLoader loader;
    046
    047        // relative to loader
    048        protected int[] begSlice = null; // slice begin
    049        protected int[] delSlice = null; // slice delta
    050        /**
    051         * @since 2.2
    052         */
    053        protected int[] sShape   = null; // sliced shape
    054
    055        /**
    056         * @since 2.2
    057         */
    058        protected int[] padding = null; // differences in shape from original (or sliced) shape
    059        protected int[] map; // transposition map (same length as current shape)
    060
    061        /**
    062         * Create a lazy dataset
    063         * @param loader
    064         * @param name
    065         * @param elements
    066         * @param clazz dataset interface
    067         * @param shape
    068         * @since 2.3
    069         */
    070        public LazyDataset(ILazyLoader loader, String name, int elements, Class<? extends Dataset> clazz, int... shape) {
    071                this.loader = loader;
    072                this.name = name;
    073                this.isize = elements;
    074                this.clazz = clazz;
    075                this.shape = shape.clone();
    076                this.oShape = this.shape;
    077                try {
    078                        size = ShapeUtils.calcLongSize(shape);
    079                } catch (IllegalArgumentException e) {
    080                        size = Long.MAX_VALUE; // this indicates that the entire dataset cannot be read in!
    081                }
    082        }
    083
    084        /**
    085         * Create a lazy dataset
    086         * @param loader
    087         * @param name
    088         * @param clazz dataset interface
    089         * @param shape
    090         * @since 2.3
    091         */
    092        public LazyDataset(ILazyLoader loader, String name, Class<? extends Dataset> clazz, int... shape) {
    093                this(loader, name, 1, clazz, shape);
    094        }
    095
    096        /**
    097         * Create a lazy dataset
    098         * @param name
    099         * @param dtype dataset type
    100         * @param elements
    101         * @param shape
    102         * @param loader
    103         * @deprecated Use {@link #LazyDataset(ILazyLoader, String, int, Class, int[])}
    104         */
    105        @Deprecated
    106        public LazyDataset(String name, int dtype, int elements, int[] shape, ILazyLoader loader) {
    107                this(loader, name, elements, DTypeUtils.getInterface(dtype), shape);
    108        }
    109
    110        /**
    111         * Create a lazy dataset
    112         * @param name
    113         * @param dtype dataset type
    114         * @param shape
    115         * @param loader
    116         * @deprecated Use {@link #LazyDataset(ILazyLoader, String, int, Class, int[])}
    117         */
    118        @Deprecated
    119        public LazyDataset(String name, int dtype, int[] shape, ILazyLoader loader) {
    120                this(name, dtype, 1, shape, loader);
    121        }
    122
    123        LazyDataset(LazyDataset other) {
    124                name  = other.name;
    125                shape = other.shape.clone();
    126                metadata  = other.copyMetadata();
    127                oMetadata = other.oMetadata;
    128                oShape = other.oShape;
    129                size   = other.size;
    130                clazz  = other.clazz;
    131                isize  = other.isize;
    132                loader = other.loader;
    133
    134                begSlice = other.begSlice;
    135                delSlice = other.delSlice;
    136                sShape   = other.sShape;
    137                padding  = other.padding;
    138                map      = other.map;
    139        }
    140
    141        /**
    142         * Create a lazy dataset based on in-memory data (handy for testing)
    143         * @param dataset
    144         */
    145        public static LazyDataset createLazyDataset(final Dataset dataset) {
    146                return new LazyDataset(dataset.getName(), dataset.getDType(), dataset.getElementsPerItem(), dataset.getShapeRef(),
    147                new ILazyLoader() {
    148                        private static final long serialVersionUID = -6725268922780517523L;
    149
    150                        final Dataset d = dataset;
    151                        @Override
    152                        public boolean isFileReadable() {
    153                                return true;
    154                        }
    155
    156                        @Override
    157                        public Dataset getDataset(IMonitor mon, SliceND slice) throws IOException {
    158                                return d.getSlice(mon, slice);
    159                        }
    160                });
    161        }
    162
    163        /**
    164         * Can return -1 for unknown
    165         */
    166        @Override
    167        public int getDType() {
    168                return DTypeUtils.getDType(clazz);
    169        }
    170
    171        /**
    172         * @return dataset interface that supports element class and number of elements
    173         * @since 2.3
    174         */
    175        public Class<? extends Dataset> getInterface() {
    176                return clazz;
    177        }
    178
    179        /**
    180         * Can return -1 for unknown
    181         */
    182        @Override
    183        public int getElementsPerItem() {
    184                return isize;
    185        }
    186
    187        @Override
    188        public int getSize() {
    189                return (int) size;
    190        }
    191
    192        @Override
    193        public String toString() {
    194                StringBuilder out = new StringBuilder();
    195
    196                if (name != null && name.length() > 0) {
    197                        out.append("Lazy dataset '");
    198                        out.append(name);
    199                        out.append("' has shape [");
    200                } else {
    201                        out.append("Lazy dataset shape is [");
    202                }
    203                int rank = shape == null ? 0 : shape.length;
    204
    205                if (rank > 0 && shape[0] >= 0) {
    206                        out.append(shape[0]);
    207                }
    208                for (int i = 1; i < rank; i++) {
    209                        out.append(", " + shape[i]);
    210                }
    211                out.append(']');
    212
    213                return out.toString();
    214        }
    215
    216        @Override
    217        public int hashCode() {
    218                final int prime = 31;
    219                int result = super.hashCode();
    220                result = prime * result + Arrays.hashCode(oShape);
    221                result = prime * result + (int) (size ^ (size >>> 32));
    222                result = prime * result + clazz.hashCode();
    223                result = prime * result + isize;
    224                result = prime * result + ((loader == null) ? 0 : loader.hashCode());
    225                result = prime * result + Arrays.hashCode(begSlice);
    226                result = prime * result + Arrays.hashCode(delSlice);
    227                result = prime * result + Arrays.hashCode(sShape);
    228                result = prime * result + Arrays.hashCode(padding);
    229                result = prime * result + Arrays.hashCode(map);
    230                return result;
    231        }
    232
    233        @Override
    234        public boolean equals(Object obj) {
    235                if (this == obj) {
    236                        return true;
    237                }
    238                if (!super.equals(obj)) {
    239                        return false;
    240                }
    241
    242                LazyDataset other = (LazyDataset) obj;
    243                if (!Arrays.equals(oShape, other.oShape)) {
    244                        return false;
    245                }
    246                if (size != other.size) {
    247                        return false;
    248                }
    249                if (!clazz.equals(other.clazz)) {
    250                        return false;
    251                }
    252                if (isize != other.isize) {
    253                        return false;
    254                }
    255
    256                if (loader != other.loader) {
    257                        return false;
    258                }
    259
    260                if (!Arrays.equals(begSlice, other.begSlice)) {
    261                        return false;
    262                }
    263                if (!Arrays.equals(delSlice, other.delSlice)) {
    264                        return false;
    265                }
    266                if (!Arrays.equals(sShape, other.sShape)) {
    267                        return false;
    268                }
    269                if (!Arrays.equals(padding, other.padding)) {
    270                        return false;
    271                }
    272                if (!Arrays.equals(map, other.map)) {
    273                        return false;
    274                }
    275
    276                return true;
    277        }
    278
    279        @Override
    280        public LazyDataset clone() {
    281                return new LazyDataset(this);
    282        }
    283
    284        @Override
    285        public void setShape(int... shape) {
    286                setShapeInternal(shape.clone());
    287        }
    288
    289        @Override
    290        public LazyDataset squeezeEnds() {
    291                setShapeInternal(ShapeUtils.squeezeShape(shape, true));
    292                return this;
    293        }
    294
    295        @Override
    296        public Dataset getSlice(int[] start, int[] stop, int[] step) throws DatasetException {
    297                return getSlice(null, start, stop, step);
    298        }
    299
    300        @Override
    301        public Dataset getSlice(Slice... slice) throws DatasetException {
    302                if (slice == null || slice.length == 0) {
    303                        return getSlice(null, new SliceND(shape));
    304                }
    305                return getSlice(null, new SliceND(shape, slice));
    306        }
    307
    308        @Override
    309        public Dataset getSlice(SliceND slice) throws DatasetException {
    310                return getSlice(null, slice);
    311        }
    312
    313        @Override
    314        public Dataset getSlice(IMonitor monitor, Slice... slice) throws DatasetException {
    315                if (slice == null || slice.length == 0) {
    316                        return getSlice(monitor, new SliceND(shape));
    317                }
    318                return getSlice(monitor, new SliceND(shape, slice));
    319        }
    320
    321        @Override
    322        public LazyDataset getSliceView(Slice... slice) {
    323                if (slice == null || slice.length == 0) {
    324                        return getSliceView(new SliceND(shape));
    325                }
    326                return getSliceView(new SliceND(shape, slice));
    327        }
    328
    329        /**
    330         * @param nShape
    331         */
    332        private void setShapeInternal(int... nShape) {
    333                // work out transposed (sliced) shape (instead of removing padding from current shape)
    334                if (size != 0) {
    335                        int[] pShape = calcTransposed(map, sShape == null ? oShape : sShape);
    336                        padding = ShapeUtils.calcShapePadding(pShape, nShape);
    337                }
    338
    339                if (metadata != null) {
    340                        storeMetadata(metadata, Reshapeable.class);
    341                        metadata = copyMetadata();
    342                        reshapeMetadata(shape, nShape);
    343                }
    344                shape = nShape;
    345        }
    346
    347        @Override
    348        public LazyDataset getSliceView(int[] start, int[] stop, int[] step) {
    349                return getSliceView(new SliceND(shape, start, stop, step));
    350        }
    351
    352        @Override
    353        public LazyDataset getSliceView(SliceND slice) {
    354                LazyDataset view = clone();
    355                if (slice.isAll()) {
    356                        return view;
    357                }
    358
    359                SliceND nslice = calcTrueSlice(slice);
    360                if (nslice != null) {
    361                        view.begSlice = nslice.getStart();
    362                        view.delSlice = nslice.getStep();
    363                        view.sShape = nslice.getShape();
    364                }
    365                view.shape = slice.getShape();
    366                view.size = ShapeUtils.calcLongSize(view.shape);
    367                view.storeMetadata(metadata, Sliceable.class);
    368
    369                view.sliceMetadata(true, slice);
    370                return view;
    371        }
    372
    373        @Override
    374        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws DatasetException {
    375                return getSlice(monitor, new SliceND(shape, start, stop, step));
    376        }
    377
    378        @Override
    379        public Dataset getSlice(IMonitor monitor, SliceND slice) throws DatasetException {
    380                if (loader != null && !loader.isFileReadable()) {
    381                        return null;
    382                }
    383
    384                SliceND nslice = calcTrueSlice(slice);
    385
    386                Dataset a;
    387                if (nslice == null) {
    388                        a = DatasetFactory.zeros(clazz, slice.getShape());
    389                } else {
    390                        try {
    391                                a = DatasetUtils.convertToDataset(loader.getDataset(monitor, nslice));
    392                        } catch (IOException e) {
    393                                logger.error("Problem getting {}: {}", String.format("slice %s %s %s from %s", Arrays.toString(slice.getStart()), Arrays.toString(slice.getStop()),
    394                                                                Arrays.toString(slice.getStep()), loader), e);
    395                                throw new DatasetException(e);
    396                        }
    397                }
    398                a.setName(name + AbstractDataset.BLOCK_OPEN + (nslice == null ? slice : nslice) + AbstractDataset.BLOCK_CLOSE);
    399                if (metadata != null && a instanceof LazyDatasetBase) {
    400                        LazyDatasetBase ba = (LazyDatasetBase) a;
    401                        ba.metadata = copyMetadata();
    402                        if (oMetadata != null) {
    403                                ba.restoreMetadata(oMetadata);
    404                        }
    405                        // metadata axis may be larger than data
    406                        if (nslice != null && (!nslice.isAll() || nslice.getMaxShape() != nslice.getShape())) {
    407                                ba.sliceMetadata(true, nslice);
    408                        }
    409                }
    410
    411                if (nslice != null) {
    412                        if (map != null) {
    413                                a = a.getTransposedView(map);
    414                        }
    415                        if (padding != null) {
    416                                a.setShape(slice.getShape());
    417                        }
    418                }
    419                a.addMetadata(MetadataFactory.createMetadata(OriginMetadata.class, this, nslice == null ? slice.convertToSlice() : nslice.convertToSlice(), oShape, null, name));
    420
    421                return a;
    422        }
    423
    424        @Override
    425        public LazyDataset getTransposedView(final int... axes) {
    426                LazyDataset view = clone();
    427
    428                int[] naxes = checkPermutatedAxes(shape, axes);
    429                if (naxes == null) {
    430                        return view;
    431                }
    432
    433                view.shape = calcTransposed(naxes, shape);
    434                if (view.size != 0 && padding != null) { // work out transpose by reverting effect of padding
    435                        int or = oShape.length;
    436                        int nr = shape.length;
    437                        int j = 0; // naxes index
    438                        int[] mShape = calcTransposed(map, sShape == null ? oShape : sShape); // pre-padded shape
    439                        int m = 0; // shape index
    440                        int e = -1; // index of unit dimension
    441                        final List<Integer> uaxes = new LinkedList<>();
    442                        for (int a : naxes) {
    443                                uaxes.add(a);
    444                        }
    445                        List<Integer> oList = new ArrayList<>(); // dimensions left out by padding (in order)
    446                        int np = padding.length;
    447                        for (int i = 0; i < np; i++) {
    448                                int p = padding[i];
    449                                if (p > 0) { // remove added dimensions
    450                                        for (int k = 0; k < p; k++, j++) {
    451                                                uaxes.remove((Integer) j);
    452                                        }
    453                                } else if (p == 0) { // leave alone
    454                                        if (mShape[m] == 1) { // bump up last unit dimension index
    455                                                e = m;
    456                                        }
    457                                        j++;
    458                                        m++;
    459                                } else { // add omitted dimensions to list
    460                                        p = -p;
    461                                        for (int k = 0; k < p; k++) {
    462                                                e = find(mShape, 1, e + 1);
    463                                                oList.add(e);
    464                                        }
    465                                }
    466                        }
    467
    468                        int[] omitted = new int[oList.size()];
    469                        j = 0;
    470                        for (Integer o : oList) {
    471                                omitted[j++] = o;
    472                        }
    473                        int[] used = new int[or - omitted.length]; // all dimensions not omitted in pre-padded shape
    474                        j = 0;
    475                        for (int i = 0; i < or; i++) {
    476                                if (Arrays.binarySearch(omitted, i) < 0) {
    477                                        used[j++] = i;
    478                                }
    479                        }
    480
    481                        int[] vaxes = new int[uaxes.size()];
    482                        j = 0;
    483                        for (int i = 0; i < nr; i++) { // remap dimension numbering
    484                                int l = uaxes.indexOf(i);
    485                                if (l >= 0) {
    486                                        vaxes[l] = used[j++];
    487                                }
    488                        }
    489                        int[] taxes = new int[or];
    490                        j = 0;
    491                        for (int i = 0; i < or; i++) { // reassemble map
    492                                if (Arrays.binarySearch(omitted, i) >= 0) {
    493                                        taxes[i] = i;
    494                                } else {
    495                                        taxes[i] = vaxes[j++];
    496                                }
    497                        }
    498
    499                        naxes = taxes;
    500                }
    501
    502                view.map = map == null ? naxes : calcTransposed(naxes, map);
    503                if (view.size != 0) {
    504                        // work out transposed (sliced) shape
    505                        int[] tShape = calcTransposed(view.map, sShape == null ? oShape : sShape);
    506                        try {
    507                                view.padding = ShapeUtils.calcShapePadding(tShape, view.shape);
    508                        } catch (IllegalArgumentException e) {
    509                                System.err.println(e.getMessage() + ": " + Arrays.toString(tShape) + " cf " + Arrays.toString(view.shape));
    510                        }
    511                }
    512                view.storeMetadata(metadata, Transposable.class);
    513                view.transposeMetadata(axes);
    514                return view;
    515        }
    516
    517        private static int find(int[] map, int m, int off) {
    518                for (int i = off, imax = map.length; i < imax; i++) {
    519                        if (map[i] == m) {
    520                                return i;
    521                        }
    522                }
    523                return -1;
    524        }
    525
    526        private static int[] calcTransposed(int[] map, int[] values) {
    527                if (values == null) {
    528                        return null;
    529                }
    530                int r = values.length;
    531                if (map == null || r < 2) {
    532                        return values;
    533                }
    534                int[] ovalues = new int[r];
    535                for (int i = 0; i < r; i++) {
    536                        ovalues[i] = values[map[i]];
    537                }
    538                return ovalues;
    539        }
    540
    541        /**
    542         * Calculate absolute slice
    543         * @param slice
    544         * @return true slice or null if zero-sized
    545         */
    546        protected final SliceND calcTrueSlice(SliceND slice) {
    547                /*
    548                 * Lazy dataset operations: getTransposedView (T), getSliceView (G), setShape/squeezeEnds (S+/S-):
    549                 *
    550                 *     . T sets shape, base, and map in new view
    551                 *     . G sets shape, size, begSlice and delSlice in new view
    552                 *     . S sets shape, shapePadding in current view
    553                 *
    554                 * Then getSlice needs to interpret all info to find true slice, load data, get transposition (view)
    555                 * and set shape. Therefore:
    556                 *     . S needs to update shapePadding only
    557                 *     . T needs to update shapePadding too
    558                 *     . G needs to work out true slice to update
    559                 *
    560                 * slice -> true slice
    561                 *   adjusts for shape (S^-1) then remap dimensions (T^-1)
    562                 */
    563
    564                if (slice == null) {
    565                        slice = new SliceND(shape);
    566                }
    567
    568                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
    569                        return null;
    570                }
    571
    572                int[] nshape;
    573                int[] nstart;
    574                int[] nstep;
    575
    576                int r = oShape.length;
    577                if (padding == null) {
    578                        nshape = slice.getShape();
    579                        nstart = slice.getStart();
    580                        nstep = slice.getStep();
    581                } else {
    582                        final int[] lshape = slice.getShape();
    583                        final int[] lstart = slice.getStart();
    584                        final int[] lstep  = slice.getStep();
    585
    586                        nstart = new int[r];
    587                        nstep = new int[r];
    588                        nshape = new int[r];
    589                        int i = 0;
    590                        int j = 0;
    591                        for (int p : padding) { // remove padding
    592                                if (p == 0) {
    593                                        nshape[i] = lshape[j];
    594                                        nstart[i] = lstart[j];
    595                                        nstep[i]  = lstep[j];
    596                                        i++;
    597                                        j++;
    598                                } else if (p < 0) {
    599                                        int imax = i - p;
    600                                        while (i < imax) {
    601                                                nshape[i] = 1;
    602                                                nstep[i]  = 1;
    603                                                i++;
    604                                        }
    605                                } else {
    606                                        j += p;
    607                                }
    608                        }
    609                }
    610
    611                if (map != null && r > 1) { // transpose dimensions
    612                        int[] pshape = new int[r];
    613                        int[] pstart = new int[r];
    614                        int[] pstep = new int[r];
    615                        for (int i = 0; i < r; i++) {
    616                                int m = map[i];
    617                                pshape[m] = nshape[i];
    618                                pstart[m] = nstart[i];
    619                                pstep[m]  = nstep[i];
    620                        }
    621
    622                        nshape = pshape;
    623                        nstart = pstart;
    624                        nstep  = pstep;
    625                }
    626
    627                int[] nstop = new int[r];
    628                if (begSlice != null) { // find net slice
    629                        for (int i = 0; i < r; i++) {
    630                                int b = begSlice[i];
    631                                int d = delSlice[i];
    632                                nstart[i] = b + nstart[i] * d;
    633                                int nd = nstep[i] * d;
    634                                nstep[i] = nd;
    635                                nstop[i]  = nstart[i] + (nshape[i] - 1) * nd + (nd >= 0 ? 1 : -1);
    636                        }
    637                } else {
    638                        for (int i = 0; i < r; i++) {
    639                                int d = nstep[i];
    640                                nstop[i] = nstart[i] + (nshape[i] - 1) * d + (d >= 0 ? 1 : -1);
    641                        }
    642                }
    643
    644                return createSlice(nstart, nstop, nstep);
    645        }
    646
    647        protected SliceND createSlice(int[] nstart, int[] nstop, int[] nstep) {
    648                return SliceND.createSlice(oShape, null, nstart, nstop, nstep);
    649        }
    650
    651        /**
    652         * Transform data so that it can be used in setSlice of saver
    653         * @param data
    654         * @param tslice true slice
    655         * @return data with dimensions adjusted and remapped
    656         */
    657        final IDataset transformInput(IDataset data, SliceND tslice) {
    658                if (padding != null) { // remove padding
    659                        data = data.getSliceView();
    660                        int[] nshape = tslice.getShape();
    661                        data.setShape(nshape);
    662                }
    663
    664                return map == null ? data : data.getTransposedView(map);
    665        }
    666
    667        /**
    668         * Store metadata items that has given annotation
    669         * @param origMetadata
    670         * @param aclazz
    671         */
    672        private void storeMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> origMetadata, Class<? extends Annotation> aclazz) {
    673                List<Class<? extends MetadataType>> mclazzes = findAnnotatedMetadata(aclazz);
    674                if (mclazzes.size() == 0) {
    675                        return;
    676                }
    677
    678                if (oMetadata == null) {
    679                        oMetadata = new HashMap<Class<? extends MetadataType>, List<MetadataType>>();
    680                }
    681                for (Class<? extends MetadataType> mc : mclazzes) {
    682                        if (oMetadata.containsKey(mc)) {
    683                                continue; // do not overwrite original
    684                        }
    685
    686                        List<MetadataType> l = origMetadata.get(mc);
    687                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
    688                        for (MetadataType m : l) {
    689                                nl.add(m.clone());
    690                        }
    691                        oMetadata.put(mc, nl);
    692                }
    693        }
    694
    695        @SuppressWarnings("unchecked")
    696        private List<Class<? extends MetadataType>> findAnnotatedMetadata(Class<? extends Annotation> aclazz) {
    697                List<Class<? extends MetadataType>> mclazzes = new ArrayList<Class<? extends MetadataType>>();
    698                if (metadata == null) {
    699                        return mclazzes;
    700                }
    701
    702                for (Class<? extends MetadataType> c : metadata.keySet()) {
    703                        boolean hasAnn = false;
    704                        for (MetadataType m : metadata.get(c)) {
    705                                if (m == null) {
    706                                        continue;
    707                                }
    708
    709                                Class<? extends MetadataType> mc = m.getClass();
    710                                do { // iterate over super-classes
    711                                        for (Field f : mc.getDeclaredFields()) {
    712                                                if (f.isAnnotationPresent(aclazz)) {
    713                                                        hasAnn = true;
    714                                                        break;
    715                                                }
    716                                        }
    717                                        Class<?> sclazz = mc.getSuperclass();
    718                                        if (!MetadataType.class.isAssignableFrom(sclazz)) {
    719                                                break;
    720                                        }
    721                                        mc = (Class<? extends MetadataType>) sclazz;
    722                                } while (!hasAnn);
    723                                if (hasAnn) {
    724                                        break;
    725                                }
    726                        }
    727                        if (hasAnn) {
    728                                mclazzes.add(c);
    729                        }
    730                }
    731                return mclazzes;
    732        }
    733
    734        /**
    735         * Gets the maximum size of a slice of a dataset in a given dimension
    736         * which should normally fit in memory. Note that it might be possible
    737         * to get more in memory, this is a conservative estimate and seems to
    738         * almost always work at the size returned; providing Xmx is less than
    739         * the physical memory.
    740         *
    741         * To get more in memory increase -Xmx setting or use an expression
    742         * which calls a rolling function (like rmean) instead of slicing directly
    743         * to memory.
    744         *
    745         * @param lazySet
    746         * @param dimension
    747         * @return maximum size of dimension that can be sliced.
    748         */
    749        public static int getMaxSliceLength(ILazyDataset lazySet, int dimension) {
    750                // size in bytes of each item
    751                final double size = DTypeUtils.getItemBytes(DTypeUtils.getDTypeFromClass(lazySet.getElementClass()), lazySet.getElementsPerItem());
    752
    753                // Max in bytes takes into account our minimum requirement
    754                final double max  = Math.max(Runtime.getRuntime().totalMemory(), Runtime.getRuntime().maxMemory());
    755
    756                // Firstly if the whole dataset it likely to fit in memory, then we allow it.
    757                // Space specified in bytes per item available
    758                final double space = max/lazySet.getSize();
    759
    760                // If we have room for this whole dataset, then fine
    761                int[] shape = lazySet.getShape();
    762                if (space >= size) {
    763                        return shape[dimension];
    764                }
    765
    766                // Otherwise estimate what we can fit in, conservatively.
    767                // First get size of one slice, see it that fits, if not, still return 1
    768                double sizeOneSlice = size; // in bytes
    769                for (int dim = 0; dim < shape.length; dim++) {
    770                        if (dim == dimension) {
    771                                continue;
    772                        }
    773                        sizeOneSlice *= shape[dim];
    774                }
    775                double avail = max / sizeOneSlice;
    776                if (avail < 1) {
    777                        return 1;
    778                }
    779
    780                // We fudge this to leave some room
    781                return (int) Math.floor(avail/4d);
    782        }
    783}