    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.IOException;
    016import java.io.Serializable;
    017import java.lang.annotation.Annotation;
    018import java.lang.reflect.Field;
    019import java.util.ArrayList;
    020import java.util.Arrays;
    021import java.util.HashMap;
    022import java.util.LinkedList;
    023import java.util.List;
    024import java.util.Map;
    025import java.util.Objects;
    026
    027import org.eclipse.january.DatasetException;
    028import org.eclipse.january.IMonitor;
    029import org.eclipse.january.io.ILazyLoader;
    030import org.eclipse.january.metadata.MetadataFactory;
    031import org.eclipse.january.metadata.MetadataType;
    032import org.eclipse.january.metadata.OriginMetadata;
    033import org.eclipse.january.metadata.Reshapeable;
    034import org.eclipse.january.metadata.Sliceable;
    035import org.eclipse.january.metadata.Transposable;
    036import org.slf4j.Logger;
    037import org.slf4j.LoggerFactory;
    038
    039public class LazyDataset extends LazyDatasetBase implements Serializable, Cloneable {
    040        private static final long serialVersionUID = 2467865859867440242L;
    041
    042        private static final Logger logger = LoggerFactory.getLogger(LazyDataset.class);
    043
    044        protected Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata = null;
    045        protected int[] oShape; // original shape
    046        protected long  size;   // number of items
    047        private Class<? extends Dataset> clazz = null;
    048        protected int   isize;  // number of elements per item
    049
    050        protected ILazyLoader loader;
    051
    052        // relative to loader
    053        protected int[] begSlice = null; // slice begin
    054        protected int[] delSlice = null; // slice delta
    055        /**
    056         * @since 2.2
    057         */
    058        protected int[] sShape   = null; // sliced shape
    059
    060        /**
    061         * @since 2.2
    062         */
    063        protected int[] padding = null; // differences in shape from original (or sliced) shape
    064        protected int[] map; // transposition map (same length as current shape)
    065
    066        /**
    067         * Create a lazy dataset
    068         * @param loader lazy loader
    069         * @param name of dataset
    070         * @param elements item size
    071         * @param clazz dataset sub-interface
    072         * @param shape dataset shape
    073         * @since 2.3
    074         */
    075        public LazyDataset(ILazyLoader loader, String name, int elements, Class<? extends Dataset> clazz, int... shape) {
    076                this.loader = loader;
    077                this.name = name;
    078                this.isize = elements;
    079                this.clazz = clazz;
    080                this.shape = shape.clone();
    081                this.oShape = this.shape;
    082                try {
    083                        size = ShapeUtils.calcLongSize(shape);
    084                } catch (IllegalArgumentException e) {
    085                        size = Long.MAX_VALUE; // this indicates that the entire dataset cannot be read in!
    086                }
    087        }
    088
    089        /**
    090         * Create a lazy dataset
    091         * @param loader lazy loader
    092         * @param name of dataset
    093         * @param clazz dataset sub-interface
    094         * @param shape dataset shape
    095         * @since 2.3
    096         */
    097        public LazyDataset(ILazyLoader loader, String name, Class<? extends Dataset> clazz, int... shape) {
    098                this(loader, name, 1, clazz, shape);
    099        }
    100
    101        /**
    102         * Create a lazy dataset
    103         * @param name of dataset
    104         * @param dtype dataset type
    105         * @param elements item size
    106         * @param shape dataset shape
    107         * @param loader lazy loader
    108         * @deprecated Use {@link #LazyDataset(ILazyLoader, String, int, Class, int[])}
    109         */
    110        @Deprecated
    111        public LazyDataset(String name, int dtype, int elements, int[] shape, ILazyLoader loader) {
    112                this(loader, name, elements, DTypeUtils.getInterface(dtype), shape);
    113        }
    114
    115        /**
    116         * Create a lazy dataset
    117         * @param name of dataset
    118         * @param dtype dataset type
    119         * @param shape dataset shape
    120         * @param loader lazy loader
    121         * @deprecated Use {@link #LazyDataset(ILazyLoader, String, int, Class, int[])}
    122         */
    123        @Deprecated
    124        public LazyDataset(String name, int dtype, int[] shape, ILazyLoader loader) {
    125                this(name, dtype, 1, shape, loader);
    126        }
    127
    128        LazyDataset(LazyDataset other) {
    129                name  = other.name;
    130                shape = other.shape.clone();
    131                metadata  = other.copyMetadata();
    132                oMetadata = other.oMetadata;
    133                oShape = other.oShape;
    134                size   = other.size;
    135                clazz  = other.clazz;
    136                isize  = other.isize;
    137                loader = other.loader;
    138
    139                begSlice = other.begSlice;
    140                delSlice = other.delSlice;
    141                sShape   = other.sShape;
    142                padding  = other.padding;
    143                map      = other.map;
    144        }
    145
    146        /**
    147         * Create a lazy dataset based on in-memory data (handy for testing)
    148         * @param dataset input
    149         * @return lazy dataset
    150         */
    151        public static LazyDataset createLazyDataset(final Dataset dataset) {
    152                return new LazyDataset(new ILazyLoader() {
    153                        private static final long serialVersionUID = -6725268922780517523L;
    154
    155                        final Dataset d = dataset;
    156
    157                        @Override
    158                        public boolean isFileReadable() {
    159                                return true;
    160                        }
    161
    162                        @Override
    163                        public Dataset getDataset(IMonitor mon, SliceND slice) throws IOException {
    164                                return d.getSlice(mon, slice);
    165                        }
    166                }, dataset.getName(), dataset.getElementsPerItem(), dataset.getClass(), dataset.getShapeRef());
    167        }
    168
    169        @Override
    170        public Class<?> getElementClass() {
    171                return InterfaceUtils.getElementClass(clazz);
    172        }
    173
    174        /**
    175         * Can return -1 when dataset interface is not defined
    176         */
    177        @Override
    178        public int getDType() {
    179                return clazz == null ? -1 : DTypeUtils.getDType(clazz);
    180        }
    181
    182        /**
    183         * @return dataset interface that supports element class and number of elements. Can be null when undefined
    184         * @since 2.3
    185         */
    186        public Class<? extends Dataset> getInterface() {
    187                return clazz;
    188        }
    189
    190        /**
    191         * Set interface
    192         * @param clazz dataset sub-interface
    193         * @since 2.3
    194         */
    195        public void setInterface(Class<? extends Dataset> clazz) {
    196                this.clazz = clazz;
    197        }
    198
    199        /**
    200         * Can return -1 for unknown
    201         */
    202        @Override
    203        public int getElementsPerItem() {
    204                return isize;
    205        }
    206
    207        @Override
    208        public int getSize() {
    209                return (int) size;
    210        }
    211
    212        @Override
    213        public String toString() {
    214                StringBuilder out = new StringBuilder();
    215
    216                if (name != null && name.length() > 0) {
    217                        out.append("Lazy dataset '");
    218                        out.append(name);
    219                        out.append("' has shape [");
    220                } else {
    221                        out.append("Lazy dataset shape is [");
    222                }
    223                int rank = shape == null ? 0 : shape.length;
    224
    225                if (rank > 0 && shape[0] >= 0) {
    226                        out.append(shape[0]);
    227                }
    228                for (int i = 1; i < rank; i++) {
    229                        out.append(", " + shape[i]);
    230                }
    231                out.append(']');
    232
    233                return out.toString();
    234        }
    235
    236        @Override
    237        public int hashCode() {
    238                final int prime = 31;
    239                int result = super.hashCode();
    240                result = prime * result + Arrays.hashCode(oShape);
    241                result = prime * result + (int) (size ^ (size >>> 32));
    242                result = prime * result + Objects.hashCode(clazz);
    243                result = prime * result + isize;
    244                result = prime * result + Objects.hashCode(loader);
    245                result = prime * result + Arrays.hashCode(begSlice);
    246                result = prime * result + Arrays.hashCode(delSlice);
    247                result = prime * result + Arrays.hashCode(sShape);
    248                result = prime * result + Arrays.hashCode(padding);
    249                result = prime * result + Arrays.hashCode(map);
    250                return result;
    251        }
    252
    253        @Override
    254        public boolean equals(Object obj) {
    255                if (this == obj) {
    256                        return true;
    257                }
    258                if (!super.equals(obj)) {
    259                        return false;
    260                }
    261
    262                LazyDataset other = (LazyDataset) obj;
    263                if (!Arrays.equals(oShape, other.oShape)) {
    264                        return false;
    265                }
    266                if (size != other.size) {
    267                        return false;
    268                }
    269                if (!Objects.equals(clazz, other.clazz)) {
    270                        return false;
    271                }
    272                if (isize != other.isize) {
    273                        return false;
    274                }
    275
    276                if (loader != other.loader) {
    277                        return false;
    278                }
    279
    280                if (!Arrays.equals(begSlice, other.begSlice)) {
    281                        return false;
    282                }
    283                if (!Arrays.equals(delSlice, other.delSlice)) {
    284                        return false;
    285                }
    286                if (!Arrays.equals(sShape, other.sShape)) {
    287                        return false;
    288                }
    289                if (!Arrays.equals(padding, other.padding)) {
    290                        return false;
    291                }
    292                if (!Arrays.equals(map, other.map)) {
    293                        return false;
    294                }
    295
    296                return true;
    297        }
    298
    299        @Override
    300        public LazyDataset clone() {
    301                return new LazyDataset(this);
    302        }
    303
    304        @Override
    305        public void setShape(int... shape) {
    306                setShapeInternal(shape.clone());
    307        }
    308
    309        @Override
    310        public LazyDataset squeezeEnds() {
    311                setShapeInternal(ShapeUtils.squeezeShape(shape, true));
    312                return this;
    313        }
    314
    315        @Override
    316        public Dataset getSlice(int[] start, int[] stop, int[] step) throws DatasetException {
    317                return getSlice(null, start, stop, step);
    318        }
    319
    320        @Override
    321        public Dataset getSlice(Slice... slice) throws DatasetException {
    322                if (slice == null || slice.length == 0) {
    323                        return internalGetSlice(null, new SliceND(shape));
    324                }
    325                return internalGetSlice(null, new SliceND(shape, slice));
    326        }
    327
    328        @Override
    329        public Dataset getSlice(SliceND slice) throws DatasetException {
    330                return getSlice(null, slice);
    331        }
    332
    333        @Override
    334        public Dataset getSlice(IMonitor monitor, Slice... slice) throws DatasetException {
    335                if (slice == null || slice.length == 0) {
    336                        return internalGetSlice(monitor, new SliceND(shape));
    337                }
    338                return internalGetSlice(monitor, new SliceND(shape, slice));
    339        }
    340
    341        @Override
    342        public LazyDataset getSliceView(Slice... slice) {
    343                if (slice == null || slice.length == 0) {
    344                        return internalGetSliceView(new SliceND(shape));
    345                }
    346                return internalGetSliceView(new SliceND(shape, slice));
    347        }
    348
    349        /**
    350         * @param nShape
    351         */
    352        private void setShapeInternal(int... nShape) {
    353                // work out transposed (sliced) shape (instead of removing padding from current shape)
    354                if (size != 0) {
    355                        int[] pShape = calcTransposed(map, sShape == null ? oShape : sShape);
    356                        padding = ShapeUtils.calcShapePadding(pShape, nShape);
    357                }
    358
    359                if (metadata != null) {
    360                        storeMetadata(metadata, Reshapeable.class);
    361                        metadata = copyMetadata();
    362                        reshapeMetadata(shape, nShape);
    363                }
    364                shape = nShape;
    365        }
    366
    367        @Override
    368        public LazyDataset getSliceView(int[] start, int[] stop, int[] step) {
    369                return internalGetSliceView(new SliceND(shape, start, stop, step));
    370        }
    371
    372        @Override
    373        public LazyDataset getSliceView(SliceND slice) {
    374                if (slice != null) {
    375                        checkSliceND(slice);
    376                }
    377                return internalGetSliceView(slice);
    378        }
    379
    380        protected LazyDataset internalGetSliceView(SliceND slice) {
    381                LazyDataset view = clone();
    382                if (slice == null || slice.isAll()) {
    383                        return view;
    384                }
    385
    386                SliceND nslice = calcTrueSlice(slice);
    387                if (nslice != null) {
    388                        view.begSlice = nslice.getStart();
    389                        view.delSlice = nslice.getStep();
    390                        view.sShape = nslice.getShape();
    391                }
    392                view.shape = slice.getShape();
    393                view.size = ShapeUtils.calcLongSize(view.shape);
    394                view.storeMetadata(metadata, Sliceable.class);
    395
    396                view.sliceMetadata(true, slice);
    397                return view;
    398        }
    399
    400        @Override
    401        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws DatasetException {
    402                return internalGetSlice(monitor, new SliceND(shape, start, stop, step));
    403        }
    404
    405        @Override
    406        public Dataset getSlice(IMonitor monitor, SliceND slice) throws DatasetException {
    407                if (slice != null) {
    408                        checkSliceND(slice);
    409                }
    410                return internalGetSlice(monitor, slice);
    411        }
    412
    413        protected Dataset internalGetSlice(IMonitor monitor, SliceND slice) throws DatasetException {
    414                if (loader != null && !loader.isFileReadable()) {
    415                        return null;
    416                }
    417
    418                SliceND nslice = calcTrueSlice(slice);
    419
    420                Dataset a;
    421                if (nslice == null) {
    422                        a = DatasetFactory.zeros(clazz == null ? DoubleDataset.class : clazz, slice == null ? shape : slice.getShape());
    423                } else {
    424                        try {
    425                                a = DatasetUtils.convertToDataset(loader.getDataset(monitor, nslice));
    426                        } catch (IOException e) {
    427                                logger.error("Problem getting {}: {}", slice == null ? "all" : String.format("slice %s %s %s from %s", Arrays.toString(slice.getStart()), Arrays.toString(slice.getStop()),
    428                                                                Arrays.toString(slice.getStep()), loader), e);
    429                                throw new DatasetException(e);
    430                        }
    431                }
    432                a.setName(name + AbstractDataset.BLOCK_OPEN + (nslice == null ?
    433                                (slice == null ? AbstractDataset.ELLIPSIS : slice) : nslice) + AbstractDataset.BLOCK_CLOSE);
    434                if (metadata != null && a instanceof LazyDatasetBase) {
    435                        LazyDatasetBase ba = (LazyDatasetBase) a;
    436                        ba.metadata = copyMetadata();
    437                        if (oMetadata != null) {
    438                                ba.restoreMetadata(oMetadata);
    439                        }
    440                        // metadata axis may be larger than data
    441                        if (nslice != null && (!nslice.isAll() || nslice.getMaxShape() != nslice.getShape())) {
    442                                ba.sliceMetadata(true, nslice);
    443                        }
    444                }
    445
    446                if (nslice != null) {
    447                        if (map != null) {
    448                                a = a.getTransposedView(map);
    449                        }
    450                        if (padding != null) {
    451                                a.setShape(slice == null ? shape : slice.getShape());
    452                        }
    453                }
    454                a.addMetadata(MetadataFactory.createMetadata(OriginMetadata.class, this, nslice == null ?
    455                                (slice == null ? null : slice.convertToSlice()) : nslice.convertToSlice(), oShape, null, name));
    456
    457                if (clazz == null) {
    458                        clazz = a.getClass();
    459                }
    460                return a;
    461        }
    462
    463        @Override
    464        public LazyDataset getTransposedView(final int... axes) {
    465                LazyDataset view = clone();
    466
    467                int[] naxes = checkPermutatedAxes(shape, axes);
    468                if (naxes == null) {
    469                        return view;
    470                }
    471
    472                view.shape = calcTransposed(naxes, shape);
    473                if (view.size != 0 && padding != null) { // work out transpose by reverting effect of padding
    474                        int or = oShape.length;
    475                        int nr = shape.length;
    476                        int j = 0; // naxes index
    477                        int[] mShape = calcTransposed(map, sShape == null ? oShape : sShape); // pre-padded shape
    478                        int m = 0; // shape index
    479                        int e = -1; // index of unit dimension
    480                        final List<Integer> uaxes = new LinkedList<>();
    481                        for (int a : naxes) {
    482                                uaxes.add(a);
    483                        }
    484                        List<Integer> oList = new ArrayList<>(); // dimensions left out by padding (in order)
    485                        int np = padding.length;
    486                        for (int i = 0; i < np; i++) {
    487                                int p = padding[i];
    488                                if (p > 0) { // remove added dimensions
    489                                        for (int k = 0; k < p; k++, j++) {
    490                                                uaxes.remove((Integer) j);
    491                                        }
    492                                } else if (p == 0) { // leave alone
    493                                        if (mShape[m] == 1) { // bump up last unit dimension index
    494                                                e = m;
    495                                        }
    496                                        j++;
    497                                        m++;
    498                                } else { // add omitted dimensions to list
    499                                        p = -p;
    500                                        for (int k = 0; k < p; k++) {
    501                                                e = find(mShape, 1, e + 1);
    502                                                oList.add(e);
    503                                        }
    504                                }
    505                        }
    506
    507                        int[] omitted = new int[oList.size()];
    508                        j = 0;
    509                        for (Integer o : oList) {
    510                                omitted[j++] = o;
    511                        }
    512                        int[] used = new int[or - omitted.length]; // all dimensions not omitted in pre-padded shape
    513                        j = 0;
    514                        for (int i = 0; i < or; i++) {
    515                                if (Arrays.binarySearch(omitted, i) < 0) {
    516                                        used[j++] = i;
    517                                }
    518                        }
    519
    520                        int[] vaxes = new int[uaxes.size()];
    521                        j = 0;
    522                        for (int i = 0; i < nr; i++) { // remap dimension numbering
    523                                int l = uaxes.indexOf(i);
    524                                if (l >= 0) {
    525                                        vaxes[l] = used[j++];
    526                                }
    527                        }
    528                        int[] taxes = new int[or];
    529                        j = 0;
    530                        for (int i = 0; i < or; i++) { // reassemble map
    531                                if (Arrays.binarySearch(omitted, i) >= 0) {
    532                                        taxes[i] = i;
    533                                } else {
    534                                        taxes[i] = vaxes[j++];
    535                                }
    536                        }
    537
    538                        naxes = taxes;
    539                }
    540
    541                view.map = map == null ? naxes : calcTransposed(naxes, map);
    542                if (view.size != 0) {
    543                        // work out transposed (sliced) shape
    544                        int[] tShape = calcTransposed(view.map, sShape == null ? oShape : sShape);
    545                        try {
    546                                view.padding = ShapeUtils.calcShapePadding(tShape, view.shape);
    547                        } catch (IllegalArgumentException e) {
    548                                System.err.println(e.getMessage() + ": " + Arrays.toString(tShape) + " cf " + Arrays.toString(view.shape));
    549                        }
    550                }
    551                view.storeMetadata(metadata, Transposable.class);
    552                view.transposeMetadata(axes);
    553                return view;
    554        }
    555
    556        private static int find(int[] map, int m, int off) {
    557                for (int i = off, imax = map.length; i < imax; i++) {
    558                        if (map[i] == m) {
    559                                return i;
    560                        }
    561                }
    562                return -1;
    563        }
    564
    565        private static int[] calcTransposed(int[] map, int[] values) {
    566                if (values == null) {
    567                        return null;
    568                }
    569                int r = values.length;
    570                if (map == null || r < 2) {
    571                        return values;
    572                }
    573                int[] ovalues = new int[r];
    574                for (int i = 0; i < r; i++) {
    575                        ovalues[i] = values[map[i]];
    576                }
    577                return ovalues;
    578        }
    579
    580        /**
    581         * Calculate absolute slice
    582         * @param slice an n-D slice
    583         * @return true slice or null if zero-sized
    584         */
    585        protected final SliceND calcTrueSlice(SliceND slice) {
    586                /*
    587                 * Lazy dataset operations: getTransposedView (T), getSliceView (G), setShape/squeezeEnds (S+/S-):
    588                 *
    589                 *     . T sets shape, base, and map in new view
    590                 *     . G sets shape, size, begSlice and delSlice in new view
    591                 *     . S sets shape, shapePadding in current view
    592                 *
    593                 * Then getSlice needs to interpret all info to find true slice, load data, get transposition (view)
    594                 * and set shape. Therefore:
    595                 *     . S needs to update shapePadding only
    596                 *     . T needs to update shapePadding too
    597                 *     . G needs to work out true slice to update
    598                 *
    599                 * slice -> true slice
    600                 *   adjusts for shape (S^-1) then remap dimensions (T^-1)
    601                 */
    602
    603                if (slice == null) {
    604                        slice = new SliceND(shape);
    605                }
    606
    607                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
    608                        return null;
    609                }
    610
    611                int[] nshape;
    612                int[] nstart;
    613                int[] nstep;
    614
    615                int r = oShape.length;
    616                if (padding == null) {
    617                        nshape = slice.getShape();
    618                        nstart = slice.getStart();
    619                        nstep = slice.getStep();
    620                } else {
    621                        final int[] lshape = slice.getShape();
    622                        final int[] lstart = slice.getStart();
    623                        final int[] lstep  = slice.getStep();
    624
    625                        nstart = new int[r];
    626                        nstep = new int[r];
    627                        nshape = new int[r];
    628                        int i = 0;
    629                        int j = 0;
    630                        for (int p : padding) { // remove padding
    631                                if (p == 0) {
    632                                        nshape[i] = lshape[j];
    633                                        nstart[i] = lstart[j];
    634                                        nstep[i]  = lstep[j];
    635                                        i++;
    636                                        j++;
    637                                } else if (p < 0) {
    638                                        int imax = i - p;
    639                                        while (i < imax) {
    640                                                nshape[i] = 1;
    641                                                nstep[i]  = 1;
    642                                                i++;
    643                                        }
    644                                } else {
    645                                        j += p;
    646                                }
    647                        }
    648                }
    649
    650                if (map != null && r > 1) { // transpose dimensions
    651                        int[] pshape = new int[r];
    652                        int[] pstart = new int[r];
    653                        int[] pstep = new int[r];
    654                        for (int i = 0; i < r; i++) {
    655                                int m = map[i];
    656                                pshape[m] = nshape[i];
    657                                pstart[m] = nstart[i];
    658                                pstep[m]  = nstep[i];
    659                        }
    660
    661                        nshape = pshape;
    662                        nstart = pstart;
    663                        nstep  = pstep;
    664                }
    665
    666                int[] nstop = new int[r];
    667                if (begSlice != null) { // find net slice
    668                        for (int i = 0; i < r; i++) {
    669                                int b = begSlice[i];
    670                                int d = delSlice[i];
    671                                nstart[i] = b + nstart[i] * d;
    672                                int nd = nstep[i] * d;
    673                                nstep[i] = nd;
    674                                nstop[i]  = nstart[i] + (nshape[i] - 1) * nd + (nd >= 0 ? 1 : -1);
    675                        }
    676                } else {
    677                        for (int i = 0; i < r; i++) {
    678                                int d = nstep[i];
    679                                nstop[i] = nstart[i] + (nshape[i] - 1) * d + (d >= 0 ? 1 : -1);
    680                        }
    681                }
    682
    683                return createSlice(nstart, nstop, nstep);
    684        }
    685
    686        protected SliceND createSlice(int[] nstart, int[] nstop, int[] nstep) {
    687                return SliceND.createSlice(oShape, null, nstart, nstop, nstep);
    688        }
    689
    690        /**
    691         * Transform data so that it can be used in setSlice of saver
    692         * @param data input
    693         * @param tslice true slice
    694         * @return data with dimensions adjusted and remapped
    695         */
    696        final IDataset transformInput(IDataset data, SliceND tslice) {
    697                if (padding != null) { // remove padding
    698                        data = data.getSliceView();
    699                        int[] nshape = tslice == null ? shape : tslice.getShape();
    700                        data.setShape(nshape);
    701                }
    702
    703                return map == null ? data : data.getTransposedView(map);
    704        }
    705
    706        /**
    707         * Store metadata items that has given annotation
    708         * @param origMetadata original metadata
    709         * @param aclazz annotation class
    710         */
    711        private void storeMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> origMetadata, Class<? extends Annotation> aclazz) {
    712                List<Class<? extends MetadataType>> mclazzes = findAnnotatedMetadata(aclazz);
    713                if (mclazzes.size() == 0) {
    714                        return;
    715                }
    716
    717                if (oMetadata == null) {
    718                        oMetadata = new HashMap<Class<? extends MetadataType>, List<MetadataType>>();
    719                }
    720                for (Class<? extends MetadataType> mc : mclazzes) {
    721                        if (oMetadata.containsKey(mc)) {
    722                                continue; // do not overwrite original
    723                        }
    724
    725                        List<MetadataType> l = origMetadata.get(mc);
    726                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
    727                        for (MetadataType m : l) {
    728                                nl.add(m.clone());
    729                        }
    730                        oMetadata.put(mc, nl);
    731                }
    732        }
    733
    734        @SuppressWarnings("unchecked")
    735        private List<Class<? extends MetadataType>> findAnnotatedMetadata(Class<? extends Annotation> aclazz) {
    736                List<Class<? extends MetadataType>> mclazzes = new ArrayList<Class<? extends MetadataType>>();
    737                if (metadata == null) {
    738                        return mclazzes;
    739                }
    740
    741                for (Class<? extends MetadataType> c : metadata.keySet()) {
    742                        boolean hasAnn = false;
    743                        for (MetadataType m : metadata.get(c)) {
    744                                if (m == null) {
    745                                        continue;
    746                                }
    747
    748                                Class<? extends MetadataType> mc = m.getClass();
    749                                do { // iterate over super-classes
    750                                        for (Field f : mc.getDeclaredFields()) {
    751                                                if (f.isAnnotationPresent(aclazz)) {
    752                                                        hasAnn = true;
    753                                                        break;
    754                                                }
    755                                        }
    756                                        Class<?> sclazz = mc.getSuperclass();
    757                                        if (!MetadataType.class.isAssignableFrom(sclazz)) {
    758                                                break;
    759                                        }
    760                                        mc = (Class<? extends MetadataType>) sclazz;
    761                                } while (!hasAnn);
    762                                if (hasAnn) {
    763                                        break;
    764                                }
    765                        }
    766                        if (hasAnn) {
    767                                mclazzes.add(c);
    768                        }
    769                }
    770                return mclazzes;
    771        }
    772
    773        /**
    774         * Gets the maximum size of a slice of a dataset in a given dimension
    775         * which should normally fit in memory. Note that it might be possible
    776         * to get more in memory, this is a conservative estimate and seems to
    777         * almost always work at the size returned; providing Xmx is less than
    778         * the physical memory.
    779         *
    780         * To get more in memory increase -Xmx setting or use an expression
    781         * which calls a rolling function (like rmean) instead of slicing directly
    782         * to memory.
    783         *
    784         * @param lazySet lazy dataset
    785         * @param dimension to slice along
    786         * @return maximum size of dimension that can be sliced.
    787         */
    788        public static int getMaxSliceLength(ILazyDataset lazySet, int dimension) {
    789                // size in bytes of each item
    790                final double size = InterfaceUtils.getItemBytes(lazySet.getElementsPerItem(), InterfaceUtils.getInterface(lazySet));
    791
    792                // Max in bytes takes into account our minimum requirement
    793                final double max  = Math.max(Runtime.getRuntime().totalMemory(), Runtime.getRuntime().maxMemory());
    794
    795                // Firstly if the whole dataset it likely to fit in memory, then we allow it.
    796                // Space specified in bytes per item available
    797                final double space = max/lazySet.getSize();
    798
    799                // If we have room for this whole dataset, then fine
    800                int[] shape = lazySet.getShape();
    801                if (space >= size) {
    802                        return shape[dimension];
    803                }
    804
    805                // Otherwise estimate what we can fit in, conservatively.
    806                // First get size of one slice, see it that fits, if not, still return 1
    807                double sizeOneSlice = size; // in bytes
    808                for (int dim = 0; dim < shape.length; dim++) {
    809                        if (dim == dimension) {
    810                                continue;
    811                        }
    812                        sizeOneSlice *= shape[dim];
    813                }
    814                double avail = max / sizeOneSlice;
    815                if (avail < 1) {
    816                        return 1;
    817                }
    818
    819                // We fudge this to leave some room
    820                return (int) Math.floor(avail/4d);
    821        }
    822}