    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.Serializable;
    016import java.util.ArrayList;
    017import java.util.List;
    018
    019import org.eclipse.january.DatasetException;
    020import org.eclipse.january.IMonitor;
    021
    022/**
    023 * Class to aggregate a set of lazy datasets and present them as a single lazy dataset where
    024 * the first position value accesses the aggregation
    025 */
    026public class AggregateDataset extends LazyDatasetBase implements ILazyDataset {
    027
    028        /**
    029         * Update this when there are any serious changes to API
    030         */
    031        private static final long serialVersionUID = -5523566223386837581L;
    032
    033        private ILazyDataset[] data = null; // array of lazy datasets
    034        private int[] map = null;    // map first dimension to index of dataset
    035        private int[] offset = null; // cumulative first dimension lengths used as slice offsets
    036        private int size;
    037        private Class<? extends Dataset> clazz = null;
    038        private int isize; // number of elements per item
    039        protected AggregateDataset base = null;
    040        private int[] sliceStart = null;
    041        private int[] sliceStep  = null;
    042
    043        /**
    044         * Calculate (possibly extended) shapes from given datasets
    045         * @param extend if true, extend rank by one
    046         * @param datasets
    047         * @return array of shapes
    048         */
    049        public static int[][] calcShapes(boolean extend, ILazyDataset... datasets) {
    050                if (datasets.length == 0)
    051                        throw new IllegalArgumentException("No datasets given");
    052
    053                int maxRank = -1;
    054                for (ILazyDataset d : datasets) {
    055                        if (d == null)
    056                                throw new IllegalArgumentException("Null dataset given");
    057
    058                        int r = d.getRank();
    059                        if (r > maxRank)
    060                                maxRank = r;
    061                }
    062
    063                if (extend)
    064                        maxRank++;
    065
    066                int[][] shapes = new int[datasets.length][];
    067                for (int j = 0; j < datasets.length; j++) {
    068                        ILazyDataset d = datasets[j];
    069                        int[] s = d.getShape();
    070                        if (s.length < maxRank) {
    071                                int[] ns = new int[maxRank];
    072                                int start = maxRank - s.length;
    073
    074                                for (int i = 0; i < start; i++) { // prepend ones as necessary
    075                                        ns[i] = 1;
    076                                }
    077                                for (int i = 0; i < s.length; i++) {
    078                                        ns[i+start] = s[i];
    079                                }
    080                                s = ns;
    081                        }
    082                        shapes[j] = s;
    083                }
    084
    085                return shapes;
    086        }
    087
    088        AggregateDataset(int itemSize, Class<? extends Dataset> clazz, int... shape) {
    089                isize = itemSize;
    090                this.shape = shape.clone();
    091                try {
    092                        size = ShapeUtils.calcSize(shape);
    093                } catch (IllegalArgumentException e) {
    094                        size = Integer.MAX_VALUE; // this indicates that the entire dataset cannot be read in!
    095                }
    096                this.clazz = clazz;
    097        }
    098
    099        AggregateDataset(int itemSize, int[] shape, int dtype) {
    100                this(itemSize, DTypeUtils.getInterface(dtype), shape);
    101        }
    102
    103        /**
    104         * Create an aggregate dataset
    105         * @param extend if true, extend rank by one
    106         * @param datasets
    107         */
    108        public AggregateDataset(boolean extend, ILazyDataset... datasets) {
    109
    110                final int[][] shapes = calcShapes(extend, datasets);
    111
    112                // check for same (sub-)shape
    113                final int[] s = shapes[0];
    114                final int axis = extend ? -1 : 0;
    115                for (int j = 1; j < shapes.length; j++) {
    116                        if (!ShapeUtils.areShapesCompatible(s, shapes[j], axis))
    117                                throw new IllegalArgumentException("Dataset '" + datasets[j].getName() + "' has wrong shape");
    118                }
    119
    120                // set shapes of datasets
    121                final int maxRank = s.length;
    122                data = new ILazyDataset[datasets.length];
    123                isize = datasets[0].getElementsPerItem();
    124                for (int j = 0; j < datasets.length; j++) {
    125                        ILazyDataset d = datasets[j];
    126                        int[] ds = d.getShape();
    127                        if (ds.length < maxRank) {
    128                                d = d.clone();
    129                                d.setShape(shapes[j]);
    130                        }
    131                        data[j] = d;
    132                        if (d.getElementsPerItem() != isize) {
    133                                throw new IllegalArgumentException("All datasets must have the same number of elements");
    134                        }
    135                }
    136
    137                // calculate new shape
    138                shape = new int[maxRank];
    139                for (int i = 1; i < shape.length; i++) {
    140                        shape[i] = s[i];
    141                }
    142                if (extend) {
    143                        shape[0] = data.length;
    144                } else {
    145                        for (int j = 0; j < datasets.length; j++) {
    146                                shape[0] += shapes[j][0];
    147                        }
    148                }
    149                try {
    150                        size = ShapeUtils.calcSize(shape);
    151                } catch (IllegalArgumentException e) {
    152                        size = Integer.MAX_VALUE; // this indicates that the entire dataset cannot be read in!
    153                }
    154
    155                // work out offsets from cumulative lengths
    156                offset = new int[data.length];
    157                int cd = 0;
    158                for (int i = 0; i < data.length; i++) {
    159                        offset[i] = cd;
    160                        cd += data[i].getShape()[0];
    161                }
    162
    163                // calculate mapping from aggregate dimension to dataset array index
    164                map = new int[shape[0]];
    165                int k = 0;
    166                for (int i = 0; i < data.length; i++) {
    167                        int jmax = data[i].getShape()[0];
    168                        for (int j = 0; j < jmax; j++)
    169                                map[k++] = i;
    170                }
    171
    172                for (ILazyDataset d : data) {
    173                        clazz = InterfaceUtils.getBestInterface(clazz, InterfaceUtils.getInterfaceFromClass(d.getElementsPerItem(), d.getElementClass()));
    174                }
    175
    176                for (ILazyDataset d : data) {
    177                        String n = d.getName();
    178                        if (n != null) {
    179                                name = n;
    180                                break;
    181                        }
    182                }
    183        }
    184
    185        @Override
    186        public Class<?> getElementClass() {
    187                return InterfaceUtils.getElementClass(clazz);
    188        }
    189
    190        @Override
    191        public int getElementsPerItem() {
    192                return isize;
    193        }
    194
    195        @Override
    196        public int getDType() {
    197                return DTypeUtils.getDType(clazz);
    198        }
    199
    200        @Override
    201        public int getSize() {
    202                return size;
    203        }
    204
    205        @Override
    206        public void setShape(int... shape) {
    207                throw new UnsupportedOperationException("Not implemented");
    208        }
    209
    210        @Override
    211        public ILazyDataset squeezeEnds() {
    212                throw new UnsupportedOperationException("Not implemented");
    213        }
    214
    215        @Override
    216        public Dataset getSlice(int[] start, int[] stop, int[] step) throws DatasetException {
    217                return getSlice(null, new SliceND(shape, start, stop, step));
    218        }
    219
    220        @Override
    221        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws DatasetException {
    222                return getSlice(monitor, new SliceND(shape, start, stop, step));
    223        }
    224
    225        @Override
    226        public Dataset getSlice(IMonitor monitor, SliceND slice) throws DatasetException {
    227                int[] start = slice.getStart();
    228                int[] stop  = slice.getStop();
    229                int[] step  = slice.getStep();
    230
    231                if (base != null) {
    232                        for (int i = 0; i < shape.length; i++) {
    233                                start[i] = sliceStart[i] + start[i] * sliceStep[i];
    234                                stop[i]  = sliceStart[i] + (stop[i] - 1) * sliceStep[i] + 1;
    235                                step[i]  = step[i] * sliceStep[i];
    236                        }
    237                        return base.getSlice(monitor, start, stop, step);
    238                }
    239
    240                // convert first dimension's slice to individual slices per stored dataset
    241                int fb = start[0];
    242                int fe = stop[0];
    243                int fs = step[0];
    244
    245                List<Dataset> sliced = new ArrayList<Dataset>();
    246                int op = fb;
    247                int p = op;
    248                ILazyDataset od = data[map[op]];
    249                ILazyDataset nd;
    250                while (p < fe) {
    251
    252                        if (monitor!=null && monitor.isCancelled()) throw new DatasetException("Slice cancelled");
    253                        nd = data[map[p]];
    254                        if (nd != od) {
    255                                start[0] = op - offset[map[op]];
    256                                stop[0] = p - offset[map[op]];
    257                                Dataset a = DatasetUtils.convertToDataset(od.getSlice(monitor, start, stop, step));
    258                                sliced.add(a.cast(clazz));
    259
    260                                od = nd;
    261                                op = p;
    262                        }
    263                        p += fs;
    264                }
    265                start[0] = op - offset[map[op]];
    266                stop[0] = p - offset[map[op]];
    267                Dataset a = DatasetUtils.convertToDataset(od.getSlice(monitor, start, stop, step));
    268                sliced.add(a.cast(clazz));
    269
    270                Dataset d = DatasetUtils.concatenate(sliced.toArray(new Dataset[0]), 0);
    271                d.setName(name);
    272                return d;
    273        }
    274
    275        @Override
    276        public Dataset getSlice(Slice... slice) throws DatasetException {
    277                return getSlice(null, slice);
    278        }
    279
    280        @Override
    281        public Dataset getSlice(SliceND slice) throws DatasetException {
    282                return getSlice(null, slice);
    283        }
    284
    285        @Override
    286        public Dataset getSlice(IMonitor monitor, Slice... slice) throws DatasetException {
    287                return getSlice(monitor, new SliceND(shape, slice));
    288        }
    289
    290        @Override
    291        public AggregateDataset getSliceView(Slice... slice) {
    292                return getSliceView(new SliceND(shape, slice));
    293        }
    294
    295        @Override
    296        public AggregateDataset getSliceView(int[] start, int[] stop, int[] step) {
    297                return getSliceView(new SliceND(shape, start, stop, step));
    298        }
    299
    300        @Override
    301        public AggregateDataset getSliceView(SliceND slice) {
    302                AggregateDataset lazy = new AggregateDataset(isize, clazz, slice.getShape());
    303                lazy.sliceStart = slice.getStart();
    304                lazy.sliceStep  = slice.getStep();
    305                lazy.name = name + "[" + slice + "]";
    306                lazy.base = base == null ? this : base;
    307                return lazy;
    308        }
    309
    310        @Override
    311        public AggregateDataset getTransposedView(int... axes) {
    312                throw new UnsupportedOperationException("Not implemented");
    313        }
    314
    315        @Override
    316        public AggregateDataset clone() {
    317                throw new UnsupportedOperationException("Not implemented");
    318        }
    319
    320        @Override
    321        public String toString() {
    322                StringBuilder out = new StringBuilder();
    323
    324                if (name != null && name.length() > 0) {
    325                        out.append("Aggregate dataset '");
    326                        out.append(name);
    327                        out.append("' has shape [");
    328                } else {
    329                        out.append("Aggregate dataset shape is [");
    330                }
    331                int rank = shape == null ? 0 : shape.length;
    332
    333                if (rank > 0 && shape[0] > 0) {
    334                        out.append(shape[0]);
    335                }
    336                for (int i = 1; i < rank; i++) {
    337                        out.append(", " + shape[i]);
    338                }
    339                out.append(']');
    340
    341                return out.toString();
    342        }
    343
    344        @Override
    345        public void setErrors(Serializable errors) {
    346                throw new RuntimeException("setLazyErrors is unimplemented for "+getClass().getSimpleName());
    347        }
    348
    349        @Override
    350        public ILazyDataset getErrors() {
    351                return null;
    352        }
    353
    354}