    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.util.Arrays;
    016
    017import org.eclipse.january.DatasetException;
    018import org.eclipse.january.IMonitor;
    019import org.eclipse.january.metadata.StatisticsMetadata;
    020import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
    021import org.slf4j.Logger;
    022import org.slf4j.LoggerFactory;
    023
    024/**
    025 * Generic container class for data that is compound in nature
    026 *
    027 * Each subclass has an array of compound types, items of this array are composed of primitive types
    028 *
    029 * Data items can be Complex, Vector, etc
    030 *
    031 */
    032public abstract class AbstractCompoundDataset extends AbstractDataset implements CompoundDataset {
    033        // pin UID to base class
    034        private static final long serialVersionUID = Dataset.serialVersionUID;
    035
    036        private static final Logger logger = LoggerFactory.getLogger(AbstractCompoundDataset.class);
    037
    038        protected int isize; // number of elements per item
    039
    040        @Override
    041        public int getElementsPerItem() {
    042                return isize;
    043        }
    044
    045        @Override
    046        protected int get1DIndex(final int i) {
    047                int n = super.get1DIndex(i);
    048                return stride == null ? isize * n : n;
    049        }
    050
    051        @Override
    052        protected int get1DIndex(final int i, final int j) {
    053                int n = super.get1DIndex(i, j);
    054                return stride == null ? isize * n : n;
    055        }
    056
    057        @Override
    058        protected int get1DIndexFromShape(final int[] n) {
    059                return isize * super.get1DIndexFromShape(n);
    060        }
    061
    062        @Override
    063        public Dataset getUniqueItems() {
    064                throw new UnsupportedOperationException("Cannot sort compound datasets");
    065        }
    066
    067        @Override
    068        public IndexIterator getIterator(final boolean withPosition) {
    069                if (stride != null) {
    070                        return base.getSize() == 1 ?
    071                                        (withPosition ? new PositionIterator(offset, shape) :  new SingleItemIterator(offset, size)) : new StrideIterator(isize, shape, stride, offset);
    072                }
    073                return withPosition ? getSliceIterator(null, null, null) :
    074                        new ContiguousIterator(size, isize);
    075        }
    076
    077        /**
    078         * Get an iterator that picks out the chosen element from all items
    079         * @param element to choose
    080         * @return an iterator
    081         */
    082        public IndexIterator getIterator(int element) {
    083                if (element < 0)
    084                        element += isize;
    085                if (element < 0 || element > isize) {
    086                        logger.error("Invalid choice of element: {}/{}", element, isize);
    087                        throw new IllegalArgumentException("Invalid choice of element: " + element + "/" + isize);
    088                }
    089
    090                final IndexIterator it;
    091                if (stride != null) {
    092                        it = base.getSize() == 1 ? new SingleItemIterator(offset + element, size) : new StrideIterator(isize, shape, stride, offset, element);
    093                } else {
    094                        it = new ContiguousIterator(size, isize, element);
    095                }
    096
    097                return it;
    098        }
    099
    100        /**
    101         * @param slice to define iterator
    102         * @return an slice iterator that operates like an IndexIterator
    103         */
    104        @Override
    105        protected IndexIterator internalGetSliceIterator(SliceND slice) {
    106                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
    107                        return new NullIterator(shape, slice.getShape());
    108                }
    109                if (stride != null) {
    110                        return new StrideIterator(isize, shape, stride, offset, slice);
    111                }
    112
    113                return new SliceIterator(shape, size, isize, slice);
    114        }
    115
    116        /**
    117         * Constructor required for serialisation.
    118         */
    119        public AbstractCompoundDataset() {
    120        }
    121
    122        @Override
    123        public boolean equals(Object obj) {
    124                if (!super.equals(obj)) {
    125                        return false;
    126                }
    127
    128                CompoundDataset other = (CompoundDataset) obj;
    129                return isize == other.getElementsPerItem();
    130        }
    131
    132        @Override
    133        public int hashCode() {
    134                return getCompoundStats().getHash(shape);
    135        }
    136
    137        @Override
    138        public CompoundDataset cast(boolean repeat, int dtype, int isize) {
    139                return (CompoundDataset) super.cast(repeat, dtype, isize);
    140        }
    141
    142        @Override
    143        public CompoundDataset cast(int dtype) {
    144                return (CompoundDataset) super.cast(dtype);
    145        }
    146
    147        @Override
    148        abstract public AbstractCompoundDataset clone();
    149
    150        @Override
    151        public CompoundDataset flatten() {
    152                return (CompoundDataset) super.flatten();
    153        }
    154
    155        @Override
    156        public CompoundDataset getBy1DIndex(IntegerDataset index) {
    157                return (CompoundDataset) super.getBy1DIndex(index);
    158        }
    159
    160        @Override
    161        public CompoundDataset getByBoolean(Dataset selection) {
    162                return (CompoundDataset) super.getByBoolean(selection);
    163        }
    164
    165        @Override
    166        public CompoundDataset getByIndexes(Object... indexes) {
    167                return (CompoundDataset) super.getByIndexes(indexes);
    168        }
    169
    170        @Override
    171        public CompoundDataset getSlice(IMonitor mon, int[] start, int[] stop, int[] step) {
    172                return (CompoundDataset) super.getSlice(mon, start, stop, step);
    173        }
    174
    175        @Override
    176        public CompoundDataset getSlice(IMonitor mon, Slice... slice) {
    177                return (CompoundDataset) super.getSlice(mon, slice);
    178        }
    179
    180        @Override
    181        public CompoundDataset getSlice(IMonitor mon, SliceND slice) {
    182                return (CompoundDataset) super.getSlice(mon, slice);
    183        }
    184
    185        @Override
    186        public CompoundDataset getSlice(int[] start, int[] stop, int[] step) {
    187                return (CompoundDataset) super.getSlice(start, stop, step);
    188        }
    189
    190        @Override
    191        public CompoundDataset getSlice(Slice... slice) {
    192                return (CompoundDataset) super.getSlice(slice);
    193        }
    194
    195        @Override
    196        public CompoundDataset getSlice(SliceND slice) {
    197                return (CompoundDataset) super.getSlice(slice);
    198        }
    199
    200        @Override
    201        abstract public AbstractCompoundDataset getSlice(SliceIterator iterator);
    202
    203        @Override
    204        public CompoundDataset getSliceView(int[] start, int[] stop, int[] step) {
    205                return (CompoundDataset) super.getSliceView(start, stop, step);
    206        }
    207
    208        @Override
    209        public CompoundDataset getSliceView(Slice... slice) {
    210                return (CompoundDataset) super.getSliceView(slice);
    211        }
    212
    213        @Override
    214        public CompoundDataset getSliceView(SliceND slice) {
    215                return (CompoundDataset) super.getSliceView(slice);
    216        }
    217
    218        @Override
    219        public CompoundDataset getTransposedView(int... axes) {
    220                return (CompoundDataset) super.getTransposedView(axes);
    221        }
    222
    223        @Override
    224        abstract public AbstractCompoundDataset getView(boolean deepCopyMetadata);
    225
    226        @Override
    227        public CompoundDataset getBroadcastView(int... broadcastShape) {
    228                return (CompoundDataset) super.getBroadcastView(broadcastShape);
    229        }
    230
    231        @Override
    232        public CompoundDataset ifloorDivide(Object o) {
    233                return (CompoundDataset) super.ifloorDivide(o);
    234        }
    235
    236        @Override
    237        public CompoundDataset reshape(int... shape) {
    238                return (CompoundDataset) super.reshape(shape);
    239        }
    240
    241        @Override
    242        public CompoundDataset setSlice(Object obj, int[] start, int[] stop, int[] step) {
    243                return (CompoundDataset) super.setSlice(obj, start, stop, step);
    244        }
    245
    246        @Override
    247        public CompoundDataset setSlice(Object object, Slice... slice) {
    248                return (CompoundDataset) super.setSlice(object, slice);
    249        }
    250
    251        @Override
    252        public CompoundDataset sort(Integer axis) {
    253                throw new UnsupportedOperationException("Cannot sort dataset");
    254        }
    255
    256        @Override
    257        public CompoundDataset squeezeEnds() {
    258                return (CompoundDataset) super.squeezeEnds();
    259        }
    260
    261        @Override
    262        public CompoundDataset squeeze() {
    263                return (CompoundDataset) super.squeeze();
    264        }
    265
    266        @Override
    267        public CompoundDataset squeeze(boolean onlyFromEnd) {
    268                return (CompoundDataset) super.squeeze(onlyFromEnd);
    269        }
    270
    271        @Override
    272        public CompoundDataset swapAxes(int axis1, int axis2) {
    273                return (CompoundDataset) super.swapAxes(axis1, axis2);
    274        }
    275
    276        @Override
    277        public synchronized CompoundDataset synchronizedCopy() {
    278                return clone();
    279        }
    280
    281        @Override
    282        public CompoundDataset transpose(int... axes) {
    283                return (CompoundDataset) super.transpose(axes);
    284        }
    285
    286        /**
    287         * @since 2.0
    288         * @return first value
    289         */
    290        abstract protected double getFirstValue();
    291
    292        abstract protected double getFirstValue(final int i);
    293
    294        abstract protected double getFirstValue(final int i, final int j);
    295
    296        abstract protected double getFirstValue(final int...pos);
    297
    298        @Override
    299        public boolean getBoolean() {
    300                return getFirstValue() != 0;
    301        }
    302
    303        @Override
    304        public boolean getBoolean(final int i) {
    305                return getFirstValue(i) != 0;
    306        }
    307
    308        @Override
    309        public boolean getBoolean(final int i, final int j) {
    310                return getFirstValue(i, j) != 0;
    311        }
    312
    313        @Override
    314        public boolean getBoolean(final int... pos) {
    315                return getFirstValue(pos) != 0;
    316        }
    317
    318        @Override
    319        public byte getByte() {
    320                return (byte) getFirstValue();
    321        }
    322
    323        @Override
    324        public byte getByte(final int i) {
    325                return (byte) getFirstValue(i);
    326        }
    327
    328        @Override
    329        public byte getByte(final int i, final int j) {
    330                return (byte) getFirstValue(i, j);
    331        }
    332
    333        @Override
    334        public byte getByte(final int... pos) {
    335                return (byte) getFirstValue(pos);
    336        }
    337
    338        @Override
    339        public short getShort() {
    340                return (short) getFirstValue();
    341        }
    342
    343        @Override
    344        public short getShort(final int i) {
    345                return (short) getFirstValue(i);
    346        }
    347
    348        @Override
    349        public short getShort(final int i, final int j) {
    350                return (short) getFirstValue(i, j);
    351        }
    352
    353        @Override
    354        public short getShort(final int... pos) {
    355                return (short) getFirstValue(pos);
    356        }
    357
    358        @Override
    359        public int getInt() {
    360                return (int) getFirstValue();
    361        }
    362
    363        @Override
    364        public int getInt(final int i) {
    365                return (int) getFirstValue(i);
    366        }
    367
    368        @Override
    369        public int getInt(final int i, final int j) {
    370                return (int) getFirstValue(i, j);
    371        }
    372
    373        @Override
    374        public int getInt(final int... pos) {
    375                return (int) getFirstValue(pos);
    376        }
    377
    378        @Override
    379        public long getLong() {
    380                return (long) getFirstValue();
    381        }
    382
    383        @Override
    384        public long getLong(final int i) {
    385                return (long) getFirstValue(i);
    386        }
    387
    388        @Override
    389        public long getLong(final int i, final int j) {
    390                return (long) getFirstValue(i, j);
    391        }
    392
    393        @Override
    394        public long getLong(final int... pos) {
    395                return (long) getFirstValue(pos);
    396        }
    397
    398        @Override
    399        public float getFloat() {
    400                return (float) getFirstValue();
    401        }
    402
    403        @Override
    404        public float getFloat(final int i) {
    405                return (float) getFirstValue(i);
    406        }
    407
    408        @Override
    409        public float getFloat(final int i, final int j) {
    410                return (float) getFirstValue(i, j);
    411        }
    412
    413        @Override
    414        public float getFloat(final int... pos) {
    415                return (float) getFirstValue(pos);
    416        }
    417
    418        @Override
    419        public double getDouble() {
    420                return getFirstValue();
    421        }
    422
    423        @Override
    424        public double getDouble(final int i) {
    425                return getFirstValue(i);
    426        }
    427
    428        @Override
    429        public double getDouble(final int i, final int j) {
    430                return getFirstValue(i, j);
    431        }
    432
    433        @Override
    434        public double getDouble(final int... pos) {
    435                return getFirstValue(pos);
    436        }
    437
    438        @Override
    439        public void getDoubleArray(final double[] darray) {
    440                getDoubleArrayAbs(getFirst1DIndex(), darray);
    441        }
    442
    443        @Override
    444        public void getDoubleArray(final double[] darray, final int i) {
    445                getDoubleArrayAbs(get1DIndex(i), darray);
    446        }
    447
    448        @Override
    449        public void getDoubleArray(final double[] darray, final int i, final int j) {
    450                getDoubleArrayAbs(get1DIndex(i, j), darray);
    451        }
    452
    453        @Override
    454        public void getDoubleArray(final double[] darray, final int... pos) {
    455                getDoubleArrayAbs(get1DIndex(pos), darray);
    456        }
    457
    458        /**
    459         * @return statistics metadata
    460         * @since 2.0
    461         */
    462        @SuppressWarnings("unchecked")
    463        protected StatisticsMetadata<double[]> getCompoundStats() {
    464                StatisticsMetadata<double[]> md = getFirstMetadata(StatisticsMetadata.class);
    465                if (md == null || md.isDirty(this)) {
    466                        md = new StatisticsMetadataImpl<double[]>();
    467                        md.initialize(this);
    468                        setMetadata(md);
    469                }
    470                return md;
    471        }
    472
    473        @Override
    474        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
    475                logger.error("Cannot compare compound numbers");
    476                throw new UnsupportedOperationException("Cannot compare compound numbers");
    477        }
    478
    479        @Override
    480        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
    481                logger.error("Cannot compare compound numbers");
    482                throw new UnsupportedOperationException("Cannot compare compound numbers");
    483        }
    484
    485        @Override
    486        public Number max(boolean... ignoreInvalids) {
    487                logger.error("Cannot compare compound numbers");
    488                throw new UnsupportedOperationException("Cannot compare compound numbers");
    489        }
    490
    491        @Override
    492        public CompoundDataset max(int axis, boolean... ignoreInvalids) {
    493                logger.error("Cannot compare compound numbers");
    494                throw new UnsupportedOperationException("Cannot compare compound numbers");
    495        }
    496
    497        @Override
    498        public Number min(boolean... ignoreInvalids) {
    499                logger.error("Cannot compare compound numbers");
    500                throw new UnsupportedOperationException("Cannot compare compound numbers");
    501        }
    502
    503        @Override
    504        public CompoundDataset min(int axis, boolean... ignoreInvalids) {
    505                logger.error("Cannot compare compound numbers");
    506                throw new UnsupportedOperationException("Cannot compare compound numbers");
    507        }
    508
    509
    510        @Override
    511        public int[] maxPos(boolean... ignoreNaNs) {
    512                logger.error("Cannot compare compound numbers");
    513                throw new UnsupportedOperationException("Cannot compare compound numbers");
    514        }
    515
    516        @Override
    517        public int[] minPos(boolean... ignoreNaNs) {
    518                logger.error("Cannot compare compound numbers");
    519                throw new UnsupportedOperationException("Cannot compare compound numbers");
    520        }
    521
    522        @Override
    523        public CompoundDataset peakToPeak(int axis, boolean... ignoreInvalids) {
    524                logger.error("Cannot compare compound numbers");
    525                throw new UnsupportedOperationException("Cannot compare compound numbers");
    526        }
    527
    528        @Override
    529        public double[] maxItem() {
    530                return getCompoundStats().getMaximum();
    531        }
    532
    533        @Override
    534        public double[] minItem() {
    535                return getCompoundStats().getMinimum();
    536        }
    537
    538        @Override
    539        public Object mean(boolean... ignoreInvalids) {
    540                return getCompoundStats().getMean();
    541        }
    542
    543        @Override
    544        public CompoundDataset mean(int axis, boolean... ignoreInvalids) {
    545                return (CompoundDataset) super.mean(axis, ignoreInvalids);
    546        }
    547
    548        @Override
    549        public CompoundDataset product(int axis, boolean... ignoreInvalids) {
    550                return (CompoundDataset) super.product(axis, ignoreInvalids);
    551        }
    552
    553        @Override
    554        public CompoundDataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
    555                return (CompoundDataset) super.rootMeanSquare(axis, ignoreInvalids);
    556        }
    557
    558        @Override
    559        public CompoundDataset stdDeviation(int axis) {
    560                return (CompoundDataset) super.stdDeviation(axis, false);
    561        }
    562
    563        @Override
    564        public CompoundDataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    565                return (CompoundDataset) super.stdDeviation(axis, isWholePopulation, ignoreInvalids);
    566        }
    567
    568        @Override
    569        public Object sum(boolean... ignoreInvalids) {
    570                return getCompoundStats().getSum();
    571        }
    572
    573        @Override
    574        public CompoundDataset sum(int axis, boolean... ignoreInvalids) {
    575                return (CompoundDataset) super.sum(axis, ignoreInvalids);
    576        }
    577
    578        @Override
    579        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
    580                return getCompoundStats().getVariance(isWholePopulation, ignoreInvalids);
    581        }
    582
    583        @Override
    584        public CompoundDataset variance(int axis) {
    585                return (CompoundDataset) super.variance(axis, false);
    586        }
    587
    588        @Override
    589        public CompoundDataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    590                return (CompoundDataset) super.variance(axis, isWholePopulation, ignoreInvalids);
    591        }
    592
    593        @Override
    594        public double rootMeanSquare(boolean... ignoreInvalids) {
    595                StatisticsMetadata<double[]> stats = getCompoundStats();
    596
    597                double[] mean = stats.getMean(ignoreInvalids);
    598                double result = 0;
    599                for (int i = 0; i < isize; i++) {
    600                        double m = mean[i];
    601                        result += m * m;
    602                }
    603                return Math.sqrt(result + stats.getVariance(true));
    604        }
    605
    606        /**
    607         * @return error
    608         */
    609        private CompoundDataset getInternalError() {
    610                ILazyDataset led = super.getErrors();
    611                if (led == null)
    612                        return null;
    613
    614                Dataset ed = null;
    615                try {
    616                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
    617                } catch (DatasetException e) {
    618                        logger.error("Could not get data from lazy dataset", e);
    619                }
    620
    621                CompoundDataset ced; // ensure it has the same number of elements
    622                if (!(ed instanceof CompoundDataset) || ed.getElementsPerItem() != isize) {
    623                        ced = new CompoundDoubleDataset(isize, true, ed);
    624                } else {
    625                        ced = (CompoundDataset) ed;
    626                }
    627
    628                if (led != ced) {
    629                        setErrors(ced); // set back
    630                }
    631                return ced;
    632        }
    633
    634        @Override
    635        public CompoundDataset getErrors() {
    636                CompoundDataset ed = getInternalError();
    637                if (ed == null)
    638                        return null;
    639
    640                return ed.getBroadcastView(shape);
    641        }
    642
    643        @Override
    644        public double getError(final int i) {
    645                return calcError(getInternalErrorArray(true, i));
    646        }
    647
    648        @Override
    649        public double getError(final int i, final int j) {
    650                return calcError(getInternalErrorArray(true, i, j));
    651        }
    652
    653        @Override
    654        public double getError(final int... pos) {
    655                return calcError(getInternalErrorArray(true, pos));
    656        }
    657
    658        private double calcError(double[] es) {
    659                if (es == null)
    660                        return 0;
    661
    662                // assume elements are independent
    663                double e = 0;
    664                for (int k = 0; k < isize; k++) {
    665                        e += es[k];
    666                }
    667
    668                return Math.sqrt(e);
    669        }
    670
    671        @Override
    672        public double[] getErrorArray(final int i) {
    673                return getInternalErrorArray(false, i);
    674        }
    675
    676        @Override
    677        public double[] getErrorArray(final int i, final int j) {
    678                return getInternalErrorArray(false, i, j);
    679        }
    680
    681        @Override
    682        public double[] getErrorArray(final int... pos) {
    683                return getInternalErrorArray(false, pos);
    684        }
    685
    686        private Dataset getInternalError(final boolean squared) {
    687                Dataset sed = squared ? getInternalSquaredError() : getInternalError();
    688                if (sed == null)
    689                        return null;
    690
    691                return sed.getBroadcastView(shape);
    692        }
    693
    694        private double[] getInternalErrorArray(final boolean squared, final int i) {
    695                Dataset sed = getInternalError(squared);
    696                if (sed == null)
    697                        return null;
    698
    699                double[] es;
    700                if (sed instanceof CompoundDoubleDataset) {
    701                        es = ((CompoundDoubleDataset) sed).getDoubleArray(i);
    702                        if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted
    703                                Arrays.fill(es, es[0]);
    704                        }
    705                } else {
    706                        es = new double[isize];
    707                        Arrays.fill(es, ((DoubleDataset) sed).getDouble(i));
    708                }
    709                return es;
    710        }
    711
    712        private double[] getInternalErrorArray(final boolean squared, final int i, final int j) {
    713                Dataset sed = getInternalError(squared);
    714                if (sed == null)
    715                        return null;
    716
    717                double[] es;
    718                if (sed instanceof CompoundDoubleDataset) {
    719                        es = ((CompoundDoubleDataset) sed).getDoubleArray(i, j);
    720                        if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted
    721                                Arrays.fill(es, es[0]);
    722                        }
    723                } else {
    724                        es = new double[isize];
    725                        Arrays.fill(es, ((DoubleDataset) sed).getDouble(i, j));
    726                }
    727                return es;
    728        }
    729
    730        private double[] getInternalErrorArray(final boolean squared, final int... pos) {
    731                Dataset sed = getInternalError(squared);
    732                if (sed == null)
    733                        return null;
    734
    735                double[] es = new double[isize];
    736                if (sed instanceof CompoundDoubleDataset) {
    737                        es = ((CompoundDoubleDataset) sed).getDoubleArray(pos);
    738                        if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted
    739                                Arrays.fill(es, es[0]);
    740                        }
    741                } else {
    742                        es = new double[isize];
    743                        Arrays.fill(es, ((DoubleDataset) sed).getDouble(pos));
    744                }
    745                return es;
    746        }
    747}
    748