    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.util.Arrays;
    016
    017import org.eclipse.january.DatasetException;
    018import org.eclipse.january.IMonitor;
    019import org.eclipse.january.metadata.StatisticsMetadata;
    020import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
    021import org.slf4j.Logger;
    022import org.slf4j.LoggerFactory;
    023
    024/**
    025 * Generic container class for data that is compound in nature
    026 *
    027 * Each subclass has an array of compound types, items of this array are composed of primitive types
    028 *
    029 * Data items can be Complex, Vector, etc
    030 *
    031 */
    032public abstract class AbstractCompoundDataset extends AbstractDataset implements CompoundDataset {
    033        // pin UID to base class
    034        private static final long serialVersionUID = Dataset.serialVersionUID;
    035
    036        private static final Logger logger = LoggerFactory.getLogger(AbstractCompoundDataset.class);
    037
    038        protected int isize; // number of elements per item
    039
    040        @Override
    041        public int getElementsPerItem() {
    042                return isize;
    043        }
    044
    045        @Override
    046        protected int get1DIndex(final int i) {
    047                int n = super.get1DIndex(i);
    048                return stride == null ? isize * n : n;
    049        }
    050
    051        @Override
    052        protected int get1DIndex(final int i, final int j) {
    053                int n = super.get1DIndex(i, j);
    054                return stride == null ? isize * n : n;
    055        }
    056
    057        @Override
    058        protected int get1DIndexFromShape(final int[] n) {
    059                return isize * super.get1DIndexFromShape(n);
    060        }
    061
    062        @Override
    063        public Dataset getUniqueItems() {
    064                throw new UnsupportedOperationException("Cannot sort compound datasets");
    065        }
    066
    067        @Override
    068        public IndexIterator getIterator(final boolean withPosition) {
    069                if (stride != null) {
    070                        return base.getSize() == 1 ?
    071                                        (withPosition ? new PositionIterator(offset, shape) :  new SingleItemIterator(offset, size)) : new StrideIterator(isize, shape, stride, offset);
    072                }
    073                return withPosition ? getSliceIterator(null, null, null) :
    074                        new ContiguousIterator(size, isize);
    075        }
    076
    077        /**
    078         * Get an iterator that picks out the chosen element from all items
    079         * @param element to choose
    080         * @return an iterator
    081         */
    082        public IndexIterator getIterator(int element) {
    083                if (element < 0)
    084                        element += isize;
    085                if (element < 0 || element > isize) {
    086                        logger.error("Invalid choice of element: {}/{}", element, isize);
    087                        throw new IllegalArgumentException("Invalid choice of element: " + element + "/" + isize);
    088                }
    089
    090                final IndexIterator it;
    091                if (stride != null) {
    092                        it = base.getSize() == 1 ? new SingleItemIterator(offset + element, size) : new StrideIterator(isize, shape, stride, offset, element);
    093                } else {
    094                        it = new ContiguousIterator(size, isize, element);
    095                }
    096
    097                return it;
    098        }
    099
    100        @Override
    101        public IndexIterator getSliceIterator(SliceND slice) {
    102                checkSliceND(slice);
    103                return internalGetSliceIterator(slice);
    104        }
    105
    106        /**
    107         * @param slice to define iterator
    108         * @return an slice iterator that operates like an IndexIterator
    109         */
    110        @Override
    111        protected IndexIterator internalGetSliceIterator(SliceND slice) {
    112                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
    113                        return new NullIterator(shape, slice.getShape());
    114                }
    115                if (stride != null) {
    116                        return new StrideIterator(isize, shape, stride, offset, slice);
    117                }
    118
    119                return new SliceIterator(shape, size, isize, slice);
    120        }
    121
    122        /**
    123         * Constructor required for serialisation.
    124         */
    125        public AbstractCompoundDataset() {
    126        }
    127
    128        @Override
    129        public boolean equals(Object obj) {
    130                if (!super.equals(obj)) {
    131                        return false;
    132                }
    133
    134                CompoundDataset other = (CompoundDataset) obj;
    135                return isize == other.getElementsPerItem();
    136        }
    137
    138        @Override
    139        public int hashCode() {
    140                return getCompoundStats().getHash(shape);
    141        }
    142
    143        @Override
    144        public CompoundDataset cast(boolean repeat, int dtype, int isize) {
    145                return (CompoundDataset) super.cast(repeat, dtype, isize);
    146        }
    147
    148        @Override
    149        public CompoundDataset cast(int dtype) {
    150                return (CompoundDataset) super.cast(dtype);
    151        }
    152
    153        @Override
    154        abstract public AbstractCompoundDataset clone();
    155
    156        @Override
    157        public CompoundDataset flatten() {
    158                return (CompoundDataset) super.flatten();
    159        }
    160
    161        @Override
    162        public CompoundDataset getBy1DIndex(IntegerDataset index) {
    163                return (CompoundDataset) super.getBy1DIndex(index);
    164        }
    165
    166        @Override
    167        public CompoundDataset getByBoolean(Dataset selection) {
    168                return (CompoundDataset) super.getByBoolean(selection);
    169        }
    170
    171        @Override
    172        public CompoundDataset getByIndexes(Object... indexes) {
    173                return (CompoundDataset) super.getByIndexes(indexes);
    174        }
    175
    176        @Override
    177        public CompoundDataset getSlice(IMonitor mon, int[] start, int[] stop, int[] step) {
    178                return (CompoundDataset) super.getSlice(mon, start, stop, step);
    179        }
    180
    181        @Override
    182        public CompoundDataset getSlice(IMonitor mon, Slice... slice) {
    183                return (CompoundDataset) super.getSlice(mon, slice);
    184        }
    185
    186        @Override
    187        public CompoundDataset getSlice(IMonitor mon, SliceND slice) {
    188                return (CompoundDataset) super.getSlice(mon, slice);
    189        }
    190
    191        @Override
    192        public CompoundDataset getSlice(int[] start, int[] stop, int[] step) {
    193                return (CompoundDataset) super.getSlice(start, stop, step);
    194        }
    195
    196        @Override
    197        public CompoundDataset getSlice(Slice... slice) {
    198                return (CompoundDataset) super.getSlice(slice);
    199        }
    200
    201        @Override
    202        public CompoundDataset getSlice(SliceND slice) {
    203                return (CompoundDataset) super.getSlice(slice);
    204        }
    205
    206        @Override
    207        abstract public AbstractCompoundDataset getSlice(SliceIterator iterator);
    208
    209        @Override
    210        public CompoundDataset getSliceView(int[] start, int[] stop, int[] step) {
    211                return (CompoundDataset) super.getSliceView(start, stop, step);
    212        }
    213
    214        @Override
    215        public CompoundDataset getSliceView(Slice... slice) {
    216                return (CompoundDataset) super.getSliceView(slice);
    217        }
    218
    219        @Override
    220        public CompoundDataset getSliceView(SliceND slice) {
    221                return (CompoundDataset) super.getSliceView(slice);
    222        }
    223
    224        @Override
    225        public CompoundDataset getTransposedView(int... axes) {
    226                return (CompoundDataset) super.getTransposedView(axes);
    227        }
    228
    229        @Override
    230        abstract public AbstractCompoundDataset getView(boolean deepCopyMetadata);
    231
    232        @Override
    233        public CompoundDataset getBroadcastView(int... broadcastShape) {
    234                return (CompoundDataset) super.getBroadcastView(broadcastShape);
    235        }
    236
    237        @Override
    238        public CompoundDataset ifloorDivide(Object o) {
    239                return (CompoundDataset) super.ifloorDivide(o);
    240        }
    241
    242        @Override
    243        public CompoundDataset reshape(int... shape) {
    244                return (CompoundDataset) super.reshape(shape);
    245        }
    246
    247        @Override
    248        public CompoundDataset setSlice(Object obj, int[] start, int[] stop, int[] step) {
    249                return (CompoundDataset) super.setSlice(obj, start, stop, step);
    250        }
    251
    252        @Override
    253        public CompoundDataset setSlice(Object object, Slice... slice) {
    254                return (CompoundDataset) super.setSlice(object, slice);
    255        }
    256
    257        @Override
    258        public CompoundDataset sort(Integer axis) {
    259                throw new UnsupportedOperationException("Cannot sort dataset");
    260        }
    261
    262        @Override
    263        public CompoundDataset squeezeEnds() {
    264                return (CompoundDataset) super.squeezeEnds();
    265        }
    266
    267        @Override
    268        public CompoundDataset squeeze() {
    269                return (CompoundDataset) super.squeeze();
    270        }
    271
    272        @Override
    273        public CompoundDataset squeeze(boolean onlyFromEnd) {
    274                return (CompoundDataset) super.squeeze(onlyFromEnd);
    275        }
    276
    277        @Override
    278        public CompoundDataset swapAxes(int axis1, int axis2) {
    279                return (CompoundDataset) super.swapAxes(axis1, axis2);
    280        }
    281
    282        @Override
    283        public synchronized CompoundDataset synchronizedCopy() {
    284                return clone();
    285        }
    286
    287        @Override
    288        public CompoundDataset transpose(int... axes) {
    289                return (CompoundDataset) super.transpose(axes);
    290        }
    291
    292        /**
    293         * @since 2.0
    294         * @return first value
    295         */
    296        abstract protected double getFirstValue();
    297
    298        abstract protected double getFirstValue(final int i);
    299
    300        abstract protected double getFirstValue(final int i, final int j);
    301
    302        abstract protected double getFirstValue(final int...pos);
    303
    304        @Override
    305        public boolean getBoolean() {
    306                return getFirstValue() != 0;
    307        }
    308
    309        @Override
    310        public boolean getBoolean(final int i) {
    311                return getFirstValue(i) != 0;
    312        }
    313
    314        @Override
    315        public boolean getBoolean(final int i, final int j) {
    316                return getFirstValue(i, j) != 0;
    317        }
    318
    319        @Override
    320        public boolean getBoolean(final int... pos) {
    321                return getFirstValue(pos) != 0;
    322        }
    323
    324        @Override
    325        public byte getByte() {
    326                return (byte) getFirstValue();
    327        }
    328
    329        @Override
    330        public byte getByte(final int i) {
    331                return (byte) getFirstValue(i);
    332        }
    333
    334        @Override
    335        public byte getByte(final int i, final int j) {
    336                return (byte) getFirstValue(i, j);
    337        }
    338
    339        @Override
    340        public byte getByte(final int... pos) {
    341                return (byte) getFirstValue(pos);
    342        }
    343
    344        @Override
    345        public short getShort() {
    346                return (short) getFirstValue();
    347        }
    348
    349        @Override
    350        public short getShort(final int i) {
    351                return (short) getFirstValue(i);
    352        }
    353
    354        @Override
    355        public short getShort(final int i, final int j) {
    356                return (short) getFirstValue(i, j);
    357        }
    358
    359        @Override
    360        public short getShort(final int... pos) {
    361                return (short) getFirstValue(pos);
    362        }
    363
    364        @Override
    365        public int getInt() {
    366                return (int) getFirstValue();
    367        }
    368
    369        @Override
    370        public int getInt(final int i) {
    371                return (int) getFirstValue(i);
    372        }
    373
    374        @Override
    375        public int getInt(final int i, final int j) {
    376                return (int) getFirstValue(i, j);
    377        }
    378
    379        @Override
    380        public int getInt(final int... pos) {
    381                return (int) getFirstValue(pos);
    382        }
    383
    384        @Override
    385        public long getLong() {
    386                return (long) getFirstValue();
    387        }
    388
    389        @Override
    390        public long getLong(final int i) {
    391                return (long) getFirstValue(i);
    392        }
    393
    394        @Override
    395        public long getLong(final int i, final int j) {
    396                return (long) getFirstValue(i, j);
    397        }
    398
    399        @Override
    400        public long getLong(final int... pos) {
    401                return (long) getFirstValue(pos);
    402        }
    403
    404        @Override
    405        public float getFloat() {
    406                return (float) getFirstValue();
    407        }
    408
    409        @Override
    410        public float getFloat(final int i) {
    411                return (float) getFirstValue(i);
    412        }
    413
    414        @Override
    415        public float getFloat(final int i, final int j) {
    416                return (float) getFirstValue(i, j);
    417        }
    418
    419        @Override
    420        public float getFloat(final int... pos) {
    421                return (float) getFirstValue(pos);
    422        }
    423
    424        @Override
    425        public double getDouble() {
    426                return getFirstValue();
    427        }
    428
    429        @Override
    430        public double getDouble(final int i) {
    431                return getFirstValue(i);
    432        }
    433
    434        @Override
    435        public double getDouble(final int i, final int j) {
    436                return getFirstValue(i, j);
    437        }
    438
    439        @Override
    440        public double getDouble(final int... pos) {
    441                return getFirstValue(pos);
    442        }
    443
    444        @Override
    445        public void getDoubleArray(final double[] darray) {
    446                getDoubleArrayAbs(getFirst1DIndex(), darray);
    447        }
    448
    449        @Override
    450        public void getDoubleArray(final double[] darray, final int i) {
    451                getDoubleArrayAbs(get1DIndex(i), darray);
    452        }
    453
    454        @Override
    455        public void getDoubleArray(final double[] darray, final int i, final int j) {
    456                getDoubleArrayAbs(get1DIndex(i, j), darray);
    457        }
    458
    459        @Override
    460        public void getDoubleArray(final double[] darray, final int... pos) {
    461                getDoubleArrayAbs(get1DIndex(pos), darray);
    462        }
    463
    464        /**
    465         * @return statistics metadata
    466         * @since 2.0
    467         */
    468        @SuppressWarnings("unchecked")
    469        protected StatisticsMetadata<double[]> getCompoundStats() {
    470                StatisticsMetadata<double[]> md = getFirstMetadata(StatisticsMetadata.class);
    471                if (md == null || md.isDirty(this)) {
    472                        md = new StatisticsMetadataImpl<double[]>();
    473                        md.initialize(this);
    474                        setMetadata(md);
    475                }
    476                return md;
    477        }
    478
    479        @Override
    480        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
    481                logger.error("Cannot compare compound numbers");
    482                throw new UnsupportedOperationException("Cannot compare compound numbers");
    483        }
    484
    485        @Override
    486        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
    487                logger.error("Cannot compare compound numbers");
    488                throw new UnsupportedOperationException("Cannot compare compound numbers");
    489        }
    490
    491        @Override
    492        public Number max(boolean... ignoreInvalids) {
    493                logger.error("Cannot compare compound numbers");
    494                throw new UnsupportedOperationException("Cannot compare compound numbers");
    495        }
    496
    497        @Override
    498        public CompoundDataset max(int axis, boolean... ignoreInvalids) {
    499                logger.error("Cannot compare compound numbers");
    500                throw new UnsupportedOperationException("Cannot compare compound numbers");
    501        }
    502
    503        @Override
    504        public Number min(boolean... ignoreInvalids) {
    505                logger.error("Cannot compare compound numbers");
    506                throw new UnsupportedOperationException("Cannot compare compound numbers");
    507        }
    508
    509        @Override
    510        public CompoundDataset min(int axis, boolean... ignoreInvalids) {
    511                logger.error("Cannot compare compound numbers");
    512                throw new UnsupportedOperationException("Cannot compare compound numbers");
    513        }
    514
    515
    516        @Override
    517        public int[] maxPos(boolean... ignoreNaNs) {
    518                logger.error("Cannot compare compound numbers");
    519                throw new UnsupportedOperationException("Cannot compare compound numbers");
    520        }
    521
    522        @Override
    523        public int[] minPos(boolean... ignoreNaNs) {
    524                logger.error("Cannot compare compound numbers");
    525                throw new UnsupportedOperationException("Cannot compare compound numbers");
    526        }
    527
    528        @Override
    529        public CompoundDataset peakToPeak(int axis, boolean... ignoreInvalids) {
    530                logger.error("Cannot compare compound numbers");
    531                throw new UnsupportedOperationException("Cannot compare compound numbers");
    532        }
    533
    534        @Override
    535        public double[] maxItem() {
    536                return getCompoundStats().getMaximum();
    537        }
    538
    539        @Override
    540        public double[] minItem() {
    541                return getCompoundStats().getMinimum();
    542        }
    543
    544        @Override
    545        public Object mean(boolean... ignoreInvalids) {
    546                return getCompoundStats().getMean();
    547        }
    548
    549        @Override
    550        public CompoundDataset mean(int axis, boolean... ignoreInvalids) {
    551                return (CompoundDataset) super.mean(axis, ignoreInvalids);
    552        }
    553
    554        @Override
    555        public CompoundDataset product(int axis, boolean... ignoreInvalids) {
    556                return (CompoundDataset) super.product(axis, ignoreInvalids);
    557        }
    558
    559        @Override
    560        public CompoundDataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
    561                return (CompoundDataset) super.rootMeanSquare(axis, ignoreInvalids);
    562        }
    563
    564        @Override
    565        public CompoundDataset stdDeviation(int axis) {
    566                return (CompoundDataset) super.stdDeviation(axis, false);
    567        }
    568
    569        @Override
    570        public CompoundDataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    571                return (CompoundDataset) super.stdDeviation(axis, isWholePopulation, ignoreInvalids);
    572        }
    573
    574        @Override
    575        public Object sum(boolean... ignoreInvalids) {
    576                return getCompoundStats().getSum();
    577        }
    578
    579        @Override
    580        public CompoundDataset sum(int axis, boolean... ignoreInvalids) {
    581                return (CompoundDataset) super.sum(axis, ignoreInvalids);
    582        }
    583
    584        @Override
    585        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
    586                return getCompoundStats().getVariance(isWholePopulation, ignoreInvalids);
    587        }
    588
    589        @Override
    590        public CompoundDataset variance(int axis) {
    591                return (CompoundDataset) super.variance(axis, false);
    592        }
    593
    594        @Override
    595        public CompoundDataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    596                return (CompoundDataset) super.variance(axis, isWholePopulation, ignoreInvalids);
    597        }
    598
    599        @Override
    600        public double rootMeanSquare(boolean... ignoreInvalids) {
    601                StatisticsMetadata<double[]> stats = getCompoundStats();
    602
    603                double[] mean = stats.getMean(ignoreInvalids);
    604                double result = 0;
    605                for (int i = 0; i < isize; i++) {
    606                        double m = mean[i];
    607                        result += m * m;
    608                }
    609                return Math.sqrt(result + stats.getVariance(true));
    610        }
    611
    612        /**
    613         * @return error
    614         */
    615        private CompoundDataset getInternalError() {
    616                ILazyDataset led = super.getErrors();
    617                if (led == null)
    618                        return null;
    619
    620                Dataset ed = null;
    621                try {
    622                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
    623                } catch (DatasetException e) {
    624                        logger.error("Could not get data from lazy dataset", e);
    625                }
    626
    627                CompoundDataset ced; // ensure it has the same number of elements
    628                if (!(ed instanceof CompoundDataset) || ed.getElementsPerItem() != isize) {
    629                        ced = new CompoundDoubleDataset(isize, true, ed);
    630                } else {
    631                        ced = (CompoundDataset) ed;
    632                }
    633
    634                if (led != ced) {
    635                        setErrors(ced); // set back
    636                }
    637                return ced;
    638        }
    639
    640        @Override
    641        public CompoundDataset getErrors() {
    642                CompoundDataset ed = getInternalError();
    643                if (ed == null)
    644                        return null;
    645
    646                return ed.getBroadcastView(shape);
    647        }
    648
    649        @Override
    650        public double getError(final int i) {
    651                return calcError(getInternalErrorArray(true, i));
    652        }
    653
    654        @Override
    655        public double getError(final int i, final int j) {
    656                return calcError(getInternalErrorArray(true, i, j));
    657        }
    658
    659        @Override
    660        public double getError(final int... pos) {
    661                return calcError(getInternalErrorArray(true, pos));
    662        }
    663
    664        private double calcError(double[] es) {
    665                if (es == null)
    666                        return 0;
    667
    668                // assume elements are independent
    669                double e = 0;
    670                for (int k = 0; k < isize; k++) {
    671                        e += es[k];
    672                }
    673
    674                return Math.sqrt(e);
    675        }
    676
    677        @Override
    678        public double[] getErrorArray(final int i) {
    679                return getInternalErrorArray(false, i);
    680        }
    681
    682        @Override
    683        public double[] getErrorArray(final int i, final int j) {
    684                return getInternalErrorArray(false, i, j);
    685        }
    686
    687        @Override
    688        public double[] getErrorArray(final int... pos) {
    689                return getInternalErrorArray(false, pos);
    690        }
    691
    692        private Dataset getInternalError(final boolean squared) {
    693                Dataset sed = squared ? getInternalSquaredError() : getInternalError();
    694                if (sed == null)
    695                        return null;
    696
    697                return sed.getBroadcastView(shape);
    698        }
    699
    700        private double[] getInternalErrorArray(final boolean squared, final int i) {
    701                Dataset sed = getInternalError(squared);
    702                if (sed == null)
    703                        return null;
    704
    705                double[] es;
    706                if (sed instanceof CompoundDoubleDataset) {
    707                        es = ((CompoundDoubleDataset) sed).getDoubleArray(i);
    708                        if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted
    709                                Arrays.fill(es, es[0]);
    710                        }
    711                } else {
    712                        es = new double[isize];
    713                        Arrays.fill(es, ((DoubleDataset) sed).getDouble(i));
    714                }
    715                return es;
    716        }
    717
    718        private double[] getInternalErrorArray(final boolean squared, final int i, final int j) {
    719                Dataset sed = getInternalError(squared);
    720                if (sed == null)
    721                        return null;
    722
    723                double[] es;
    724                if (sed instanceof CompoundDoubleDataset) {
    725                        es = ((CompoundDoubleDataset) sed).getDoubleArray(i, j);
    726                        if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted
    727                                Arrays.fill(es, es[0]);
    728                        }
    729                } else {
    730                        es = new double[isize];
    731                        Arrays.fill(es, ((DoubleDataset) sed).getDouble(i, j));
    732                }
    733                return es;
    734        }
    735
    736        private double[] getInternalErrorArray(final boolean squared, final int... pos) {
    737                Dataset sed = getInternalError(squared);
    738                if (sed == null)
    739                        return null;
    740
    741                double[] es = new double[isize];
    742                if (sed instanceof CompoundDoubleDataset) {
    743                        es = ((CompoundDoubleDataset) sed).getDoubleArray(pos);
    744                        if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted
    745                                Arrays.fill(es, es[0]);
    746                        }
    747                } else {
    748                        es = new double[isize];
    749                        Arrays.fill(es, ((DoubleDataset) sed).getDouble(pos));
    750                }
    751                return es;
    752        }
    753}
    754