    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.Serializable;
    016import java.lang.reflect.Array;
    017import java.text.Format;
    018import java.util.Arrays;
    019import java.util.List;
    020
    021import org.eclipse.january.DatasetException;
    022import org.eclipse.january.IMonitor;
    023import org.eclipse.january.MetadataException;
    024import org.eclipse.january.metadata.ErrorMetadata;
    025import org.eclipse.january.metadata.MetadataFactory;
    026import org.eclipse.january.metadata.StatisticsMetadata;
    027import org.eclipse.january.metadata.internal.ErrorMetadataImpl;
    028import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
    029
    030/**
    031 * Generic container class for data
    032 * <p>
    033 * Each subclass has an array of primitive types, elements of this array are grouped or
    034 * compounded to make items
    035 * <p>
    036 * Data items can be boolean, integer, float, complex float, vector float, etc
    037 */
    038public abstract class AbstractDataset extends LazyDatasetBase implements Dataset {
    039        // pin UID to base class
    040        private static final long serialVersionUID = Dataset.serialVersionUID;
    041
    042        protected int size; // number of items
    043
    044        transient protected AbstractDataset base; // is null when not a view
    045        protected int[] stride; // can be null for row-major, contiguous datasets
    046        protected int offset;
    047
    048        /**
    049         * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as
    050         * wanted
    051         */
    052        protected Serializable odata = null;
    053
    054        /**
    055         * Set aliased data as base data
    056         */
    057        abstract protected void setData();
    058
    059        /**
    060         * Constructor required for serialisation.
    061         */
    062        public AbstractDataset() {
    063        }
    064
    065        @Override
    066        public synchronized Dataset synchronizedCopy() {
    067                return clone();
    068        }
    069
    070        @Override
    071        public int hashCode() {
    072                return getStats().getHash(shape);
    073        }
    074
    075        @Override
    076        abstract public AbstractDataset clone();
    077
    078        protected Format stringFormat = null;
    079
    080        @Override
    081        public void setStringFormat(Format format) {
    082                stringFormat = format;
    083        }
    084
    085        @Override
    086        public Dataset copy(final int dtype) {
    087                if (getDType() == dtype) {
    088                        return clone();
    089                }
    090                return DatasetUtils.copy(this, dtype);
    091        }
    092
    093        @Override
    094        public <T extends Dataset> T copy(Class<T> clazz) {
    095                return DatasetUtils.copy(clazz, this);
    096        }
    097
    098        @Override
    099        public Dataset cast(final int dtype) {
    100                if (getDType() == dtype) {
    101                        return this;
    102                }
    103                return DatasetUtils.cast(this, dtype);
    104        }
    105
    106        @Override
    107        public <T extends Dataset> T cast(Class<T> clazz) {
    108                return DatasetUtils.cast(clazz, this);
    109        }
    110
    111        @Override
    112        public Dataset cast(final boolean repeat, final int dtype, final int isize) {
    113                if (getDType() == dtype && getElementsPerItem() == isize) {
    114                        return this;
    115                }
    116                return DatasetUtils.cast(this, repeat, dtype, isize);
    117        }
    118
    119        @Override
    120        abstract public AbstractDataset getView(boolean deepCopyMetadata);
    121
    122        /**
    123         * Copy fields from original to view
    124         * @param orig
    125         * @param view
    126         * @param clone if true, then clone everything but bulk data
    127         * @param cloneMetadata if true, clone metadata
    128         */
    129        protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) {
    130                view.name = orig.getName();
    131                view.size = orig.getSize();
    132                view.odata = orig.getBuffer();
    133                view.offset = orig.getOffset();
    134                view.base = orig instanceof AbstractDataset ? ((AbstractDataset) orig).base : null;
    135
    136                if (clone) {
    137                        view.shape = orig.getShape();
    138                        view.stride = orig instanceof AbstractDataset && ((AbstractDataset) orig).stride != null ?
    139                                        ((AbstractDataset) orig).stride.clone() : null;
    140                } else {
    141                        view.shape = orig.getShapeRef();
    142                        view.stride = orig instanceof AbstractDataset ? ((AbstractDataset) orig).stride : null;
    143                }
    144
    145                view.metadata = getMetadataMap(orig, cloneMetadata);
    146                int odtype = orig.getDType();
    147                int vdtype = view.getDType();
    148                if (odtype != vdtype) {
    149                        view.setDirty();
    150                }
    151        }
    152
    153        @Override
    154        public IntegerDataset getIndices() {
    155                final IntegerDataset ret = DatasetUtils.indices(shape);
    156                if (getName() != null) {
    157                        ret.setName("Indices of " + getName());
    158                }
    159                return ret;
    160        }
    161
    162        @Override
    163        public Dataset getTransposedView(int... axes) {
    164                axes = checkPermutatedAxes(shape, axes);
    165
    166                AbstractDataset t = getView(true);
    167                if (axes == null || getRank() == 1)
    168                        return t;
    169
    170                int rank = shape.length;
    171                int[] tstride = new int[rank];
    172                int[] toffset = new int[1];
    173                int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset);
    174                int[] nstride = new int[rank];
    175                for (int i = 0; i < rank; i++) {
    176                        final int ax = axes[i];
    177                        nstride[i] = tstride[ax];
    178                        nshape[i] = shape[ax];
    179                }
    180                t.shape = nshape;
    181                t.stride = nstride;
    182                t.offset = toffset[0];
    183                t.base = this;
    184                t.setDirty();
    185                t.transposeMetadata(axes);
    186                return t;
    187        }
    188
    189        @Override
    190        public Dataset transpose(int... axes) {
    191                Dataset t = getTransposedView(axes);
    192                return t == null ? clone() : t.clone();
    193        }
    194
    195        @Override
    196        public Dataset swapAxes(int axis1, int axis2) {
    197                int rank = shape.length;
    198                axis1 = ShapeUtils.checkAxis(rank, axis1);
    199                axis2 = ShapeUtils.checkAxis(rank, axis2);
    200
    201                if (rank == 1 || axis1 == axis2) {
    202                        return this;
    203                }
    204
    205                int[] axes = new int[rank];
    206                for (int i = 0; i < rank; i++) {
    207                        axes[i] = i;
    208                }
    209
    210                axes[axis1] = axis2;
    211                axes[axis2] = axis1;
    212                return getTransposedView(axes);
    213        }
    214
    215        boolean isContiguous() {
    216                if (stride == null)
    217                        return true;
    218
    219                if (offset != 0)
    220                        return false;
    221
    222                int s = getElementsPerItem();
    223                for (int j = getRank() - 1; j >= 0; j--) {
    224                        if (stride[j] != s) {
    225                                return false;
    226                        }
    227                        s *= shape[j];
    228                }
    229
    230                return true;
    231        }
    232
    233        @Override
    234        public Dataset flatten() {
    235                if (!isContiguous()) { // need to make a copy if not contiguous
    236                        return clone().flatten();
    237                }
    238                return reshape(size);
    239        }
    240
    241        /**
    242         * Fill dataset from object at depth dimension
    243         * @param obj
    244         * @param depth
    245         * @param pos position
    246         */
    247        protected void fillData(Object obj, final int depth, final int[] pos) {
    248                if (obj == null) {
    249                        int dtype = getDType();
    250                        if (dtype == FLOAT32)
    251                                set(Float.NaN, pos);
    252                        else if (dtype == FLOAT64)
    253                                set(Double.NaN, pos);
    254                        return;
    255                }
    256
    257                Class<?> clazz = obj.getClass();
    258                if (obj instanceof List<?>) {
    259                        List<?> jl = (List<?>) obj;
    260                        int l = jl.size();
    261                        for (int i = 0; i < l; i++) {
    262                                Object lo = jl.get(i);
    263                                fillData(lo, depth + 1, pos);
    264                                pos[depth]++;
    265                        }
    266                        pos[depth] = 0;
    267                } else if (clazz.isArray()) {
    268                        int l = Array.getLength(obj);
    269                        if (clazz.equals(odata.getClass())) {
    270                                System.arraycopy(obj, 0, odata, get1DIndex(pos), l);
    271                        } else if (clazz.getComponentType().isPrimitive()) {
    272                                for (int i = 0; i < l; i++) {
    273                                        set(Array.get(obj, i), pos);
    274                                        pos[depth]++;
    275                                }
    276                                pos[depth] = 0;
    277                        } else {
    278                                for (int i = 0; i < l; i++) {
    279                                        fillData(Array.get(obj, i), depth + 1, pos);
    280                                        pos[depth]++;
    281                                }
    282                                pos[depth] = 0;
    283                        }
    284                } else if (obj instanceof IDataset) {
    285                        boolean[] a = new boolean[shape.length];
    286                        for (int i = depth; i < a.length; i++)
    287                                a[i] = true;
    288                        setSlice(obj, getSliceIteratorFromAxes(pos, a));
    289                } else {
    290                        set(obj, pos);
    291                }
    292        }
    293
    294        @Override
    295        public IndexIterator getIterator(final boolean withPosition) {
    296                if (stride != null) {
    297                        return base.getSize() == 1  ? (withPosition ? new PositionIterator(offset, shape) :
    298                                new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset);
    299                }
    300                if (shape == null) {
    301                        return new NullIterator(shape, shape);
    302                }
    303
    304                return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size);
    305        }
    306
    307        @Override
    308        public IndexIterator getIterator() {
    309                return getIterator(false);
    310        }
    311
    312        @Override
    313        public PositionIterator getPositionIterator(final int... axes) {
    314                return new PositionIterator(shape, axes);
    315        }
    316
    317        @Override
    318        public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) {
    319                return getSliceIterator(new SliceND(shape, start, stop, step));
    320        }
    321
    322        /**
    323         * @param slice
    324         * @return an slice iterator that operates like an IndexIterator
    325         */
    326        public IndexIterator getSliceIterator(SliceND slice) {
    327                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
    328                        return new NullIterator(shape, slice.getShape());
    329                }
    330                if (stride != null) {
    331                        return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice);
    332                }
    333                return new SliceIterator(shape, size, slice);
    334        }
    335
    336        @Override
    337        public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) {
    338                int rank = shape.length;
    339                int[] start;
    340                int[] stop = new int[rank];
    341                int[] step = new int[rank];
    342
    343                if (pos == null) {
    344                        start = new int[rank];
    345                } else if (pos.length == rank) {
    346                        start = pos.clone();
    347                } else {
    348                        throw new IllegalArgumentException("pos array length is not equal to rank of dataset");
    349                }
    350                if (axes == null) {
    351                        axes = new boolean[rank];
    352                        Arrays.fill(axes, true);
    353                } else if (axes.length != rank) {
    354                        throw new IllegalArgumentException("axes array length is not equal to rank of dataset");
    355                }
    356
    357                for (int i = 0; i < rank; i++) {
    358                        if (axes[i]) {
    359                                stop[i] = shape[i];
    360                        } else {
    361                                stop[i] = start[i] + 1;
    362                        }
    363                        step[i] = 1;
    364                }
    365                return (SliceIterator) getSliceIterator(start, stop, step);
    366        }
    367
    368        @Override
    369        public BooleanIterator getBooleanIterator(Dataset choice) {
    370                return getBooleanIterator(choice, true);
    371        }
    372
    373        @Override
    374        public BooleanIterator getBooleanIterator(Dataset choice, boolean value) {
    375                return BooleanIterator.createIterator(value, this, choice, this);
    376        }
    377
    378        @Override
    379        public Dataset getByBoolean(Dataset selection) {
    380                checkCompatibility(selection);
    381
    382                final int length = ((Number) selection.sum()).intValue();
    383                final int is = getElementsPerItem();
    384                Dataset r = DatasetFactory.zeros(is, getClass(), length);
    385                BooleanIterator biter = getBooleanIterator(selection);
    386
    387                int i = 0;
    388                while (biter.hasNext()) {
    389                        r.setObjectAbs(i, getObjectAbs(biter.index));
    390                        i += is;
    391                }
    392                return r;
    393        }
    394
    395        @Override
    396        public Dataset getBy1DIndex(IntegerDataset index) {
    397                final int is = getElementsPerItem();
    398                final Dataset r = DatasetFactory.zeros(is, getClass(), index.getShape());
    399                final IntegerIterator iter = new IntegerIterator(index, size, is);
    400
    401                int i = 0;
    402                while (iter.hasNext()) {
    403                        r.setObjectAbs(i, getObjectAbs(iter.index));
    404                        i += is;
    405                }
    406                return r;
    407        }
    408
    409        @Override
    410        public Dataset getByIndexes(final Object... indexes) {
    411                final IntegersIterator iter = new IntegersIterator(shape, indexes);
    412                final int is = getElementsPerItem();
    413                final Dataset r = DatasetFactory.zeros(is, getClass(), iter.getShape());
    414
    415                final int[] pos = iter.getPos();
    416                int i = 0;
    417                while (iter.hasNext()) {
    418                        r.setObjectAbs(i, getObject(pos));
    419                        i += is;
    420                }
    421                return r;
    422        }
    423
    424        @Override
    425        public Class<?> getElementClass() {
    426                return DTypeUtils.getElementClass(getDType());
    427        }
    428
    429        @Override
    430        public boolean hasFloatingPointElements() {
    431                Class<?> cls = getElementClass();
    432                return cls == Float.class || cls == Double.class;
    433        }
    434
    435        @Override
    436        public int getElementsPerItem() {
    437                return DTypeUtils.getElementsPerItem(getDType());
    438        }
    439
    440        @Override
    441        public int getItemBytes() {
    442                return DTypeUtils.getItemBytes(getDType(), getElementsPerItem());
    443        }
    444
    445        @Override
    446        public int getSize() {
    447                return size;
    448        }
    449
    450        @Override
    451        public int[] getShape() {
    452                // make a copy of the dimensions data, and put that out
    453                if (shape == null) {
    454                        logger.warn("Shape is null!!!");
    455                        return new int[] {};
    456                }
    457                return shape.clone();
    458        }
    459
    460        @Override
    461        public int getRank() {
    462                return shape == null ? 0 : shape.length;
    463        }
    464
    465        @Override
    466        public int getNbytes() {
    467                return getSize() * getItemBytes();
    468        }
    469
    470        /**
    471         * Check for -1 placeholder in shape and replace if necessary
    472         * @param shape
    473         * @param size
    474         */
    475        private void checkShape(int[] shape, int size) {
    476                int rank = shape.length;
    477                int found = -1;
    478                int nsize = 1;
    479                for (int i = 0; i < rank; i++) {
    480                        int d = shape[i];
    481                        if (d == -1) {
    482                                if (found == -1) {
    483                                        found = i;
    484                                } else {
    485                                        logger.error("Can only have one -1 placeholder in shape");
    486                                        throw new IllegalArgumentException("Can only have one -1 placeholder in shape");
    487                                }
    488                        } else {
    489                                nsize *= d;
    490                        }
    491                }
    492                if (found >= 0) {
    493                        shape[found] = size/nsize;
    494                } else if (nsize != size && !(rank == 0 && size == 0)) {
    495                        logger.error("New shape is not same size as old shape");
    496                        throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape));
    497                }
    498        }
    499
    500        @Override
    501        public void setShape(final int... shape) {
    502                int[] nshape = shape.clone();
    503                checkShape(nshape, size);
    504                if (Arrays.equals(this.shape, nshape)) {
    505                        return;
    506                }
    507
    508                if (stride != null) {
    509                        // the only compatible shapes are ones where new dimensions are factors of old dimensions
    510                        // or are combined adjacent old dimensions
    511                        int[] oshape = this.shape;
    512                        int orank = oshape.length;
    513                        int nrank = nshape.length;
    514                        int diff = nrank - orank;
    515                        int[] nstride = new int[nrank];
    516                        boolean ones = true;
    517                        // work forwards for broadcasting cases
    518                        for (int i = 0, j = 0; i < orank || j < nrank;) {
    519                                if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) {
    520                                        nstride[j++] = stride[i++];
    521                                } else if (j < nrank && nshape[j] == 1) {
    522                                        nstride[j++] = 0;
    523                                } else if (i < orank && oshape[i] == 1) {
    524                                        i++;
    525                                } else {
    526                                        if (j < nrank)
    527                                                j++;
    528                                        if (i < orank)
    529                                                i++;
    530                                        ones = false;
    531                                }
    532                        }
    533                        if (!ones) { // not just ones differ in shapes
    534                                int[] ostride = stride;
    535                                int ob = 0;
    536                                int oe = 1;
    537                                int nb = 0;
    538                                int ne = 1;
    539                                while (ob < orank && nb < nrank) {
    540                                        int ol = oshape[ob];
    541                                        int nl = nshape[nb];
    542
    543                                        if (nl < ol) { // find group of shape dimensions that form common size
    544                                                do { // case where new shape spreads single dimension over several dimensions
    545                                                        if (ne == nrank) {
    546                                                                break;
    547                                                        }
    548                                                        nl *= nshape[ne++];
    549                                                } while (nl < ol);
    550                                                if (nl != ol) {
    551                                                        logger.error("Subshape is incompatible with single dimension");
    552                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
    553                                                }
    554                                                int on = ne - 1;
    555                                                while (nshape[on] == 1) {
    556                                                        on--;
    557                                                }
    558
    559                                                nstride[on] = ostride[ob];
    560                                                for (int n = on - 1; n >= nb; n--) {
    561                                                        if (nshape[n] == 1)
    562                                                                continue;
    563
    564                                                        nstride[n] = nshape[on] * nstride[on];
    565                                                        on = n;
    566                                                }
    567                                        } else if (ol < nl) {
    568                                                do { // case where new shape combines several dimensions into one dimension
    569                                                        if (oe == orank) {
    570                                                                break;
    571                                                        }
    572                                                        ol *= oshape[oe++];
    573                                                } while (ol < nl);
    574                                                if (nl != ol) {
    575                                                        logger.error("Single dimension is incompatible with subshape");
    576                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
    577                                                }
    578
    579                                                int oo = oe - 1;
    580                                                while (oshape[oo] == 1) {
    581                                                        oo--;
    582                                                }
    583                                                int os = ostride[oo];
    584                                                for (int o = oo - 1; o >= ob; o--) {
    585                                                        if (oshape[o] == 1)
    586                                                                continue;
    587                                                        if (ostride[o] != oshape[oo] * ostride[oo]) {
    588                                                                logger.error("Subshape cannot be a non-contiguous view");
    589                                                                throw new IllegalArgumentException("Subshape cannot be a non-contiguous view");
    590                                                        }
    591                                                        oo = o;
    592                                                }
    593                                                nstride[nb] = os;
    594                                        } else {
    595                                                nstride[nb] = ostride[ob];
    596                                        }
    597
    598                                        ob = oe++;
    599                                        nb = ne++;
    600                                }
    601                        }
    602
    603                        stride = nstride;
    604                }
    605
    606                setDirty();
    607                if (this.shape != null && metadata != null) {
    608                        reshapeMetadata(this.shape, nshape);
    609                }
    610                this.shape = nshape;
    611        }
    612
    613        @Override
    614        public int[] getShapeRef() {
    615                return shape;
    616        }
    617
    618        @Override
    619        public int getOffset() {
    620                return offset;
    621        }
    622
    623        @Override
    624        public int[] getStrides() {
    625                return stride;
    626        }
    627
    628        @Override
    629        public Serializable getBuffer() {
    630                return odata;
    631        }
    632
    633        @Override
    634        public void overrideInternal(Serializable buffer, int... shape) {
    635                if (buffer != null) {
    636                        odata = buffer;
    637                        setData();
    638                        setDirty();
    639                }
    640
    641                if (shape != null) {
    642                        this.shape = shape.clone();
    643                        size = ShapeUtils.calcSize(this.shape);
    644                }
    645        }
    646
    647        /**
    648         * Create a stride array from dataset
    649         * @param a dataset
    650         * @param offset output offset
    651         * @return new strides
    652         */
    653        public static int[] createStrides(Dataset a, final int[] offset) {
    654                return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset);
    655        }
    656
    657        /**
    658         * Create a stride array from dataset
    659         * @param isize
    660         * @param shape
    661         * @param oStride original stride
    662         * @param oOffset original offset (only used if there is an original stride)
    663         * @param offset output offset
    664         * @return new strides
    665         */
    666        public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) {
    667                int rank = shape.length;
    668                final int[] stride;
    669                if (oStride == null) {
    670                        offset[0] = 0;
    671                        stride = new int[rank];
    672                        int s = isize;
    673                        for (int j = rank - 1; j >= 0; j--) {
    674                                stride[j] = s;
    675                                s *= shape[j];
    676                        }
    677                } else {
    678                        offset[0] = oOffset;
    679                        stride = oStride.clone();
    680                }
    681                return stride;
    682        }
    683
    684        /**
    685         * Create a stride array from slice information and a dataset
    686         * @param slice
    687         * @param a dataset
    688         * @param stride output stride
    689         * @param offset output offset
    690         * @return new shape
    691         */
    692        public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) {
    693                return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset);
    694        }
    695
    696        /**
    697         * Create a stride array from slice and dataset information
    698         * @param slice
    699         * @param isize
    700         * @param shape
    701         * @param oStride original stride
    702         * @param oOffset original offset (only used if there is an original stride)
    703         * @param stride output stride
    704         * @param offset output offset
    705         * @return new shape
    706         */
    707        public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) {
    708                int[] lstart = slice.getStart();
    709                int[] lstep = slice.getStep();
    710                int[] newShape = slice.getShape();
    711                int rank = shape.length;
    712
    713                if (oStride == null) {
    714                        int s = isize;
    715                        offset[0] = 0;
    716                        for (int j = rank - 1; j >= 0; j--) {
    717                                stride[j] = s * lstep[j];
    718                                offset[0] += s * lstart[j];
    719                                s *= shape[j];
    720                        }
    721                } else {
    722                        offset[0] = oOffset;
    723                        for (int j = 0; j < rank; j++) {
    724                                int s = oStride[j];
    725                                stride[j] = lstep[j] * s;
    726                                offset[0] += lstart[j] * s;
    727                        }
    728                }
    729
    730                return newShape;
    731        }
    732
    733        @Override
    734        public Dataset getBroadcastView(int... broadcastShape) {
    735                AbstractDataset view = getView(true);
    736
    737                if (!Arrays.equals(shape, broadcastShape)) {
    738                        List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape);
    739                        view.setShape(nShapes.get(0));
    740                        view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape);
    741                        view.base = this;
    742                        view.shape = broadcastShape.clone();
    743                        view.size = ShapeUtils.calcSize(broadcastShape);
    744                        if (view.name == null || view.name.isEmpty()) {
    745                                view.name = "Broadcast from " + Arrays.toString(shape);
    746                        } else {
    747                                view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape);
    748                        }
    749                }
    750                return view;
    751        }
    752
    753        @Override
    754        public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) {
    755                return getSliceView(new SliceND(shape, start, stop, step));
    756        }
    757
    758        @Override
    759        public Dataset getSliceView(Slice... slice) {
    760                if (slice == null || slice.length == 0) {
    761                        return getView(true);
    762                }
    763
    764                return getSliceView(new SliceND(shape, slice));
    765        }
    766
    767        /**
    768         * Get a slice of the dataset. The returned dataset is a view on a selection of items
    769         * @param slice
    770         * @return slice view
    771         */
    772        @Override
    773        public Dataset getSliceView(SliceND slice) {
    774                if (slice.isAll()) {
    775                        return getView(true);
    776                }
    777
    778                final int rank = shape.length;
    779                int[] sStride = new int[rank];
    780                int[] sOffset = new int[1];
    781
    782                int[] sShape = createStrides(slice, this, sStride, sOffset);
    783
    784                AbstractDataset s = getView(false);
    785                s.shape = sShape;
    786                s.size = ShapeUtils.calcSize(sShape);
    787                s.stride = sStride;
    788                s.offset = sOffset[0];
    789                s.base = this;
    790
    791                s.metadata = copyMetadata();
    792                s.sliceMetadata(true, slice);
    793
    794                s.setDirty();
    795                s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE);
    796
    797                return s;
    798        }
    799
    800        /**
    801         * Get flattened view index of given position
    802         * @param pos
    803         *            the integer array specifying the n-D position
    804         * @return the index on the flattened dataset
    805         */
    806        private int getFlat1DIndex(final int[] pos) {
    807                final int imax = pos.length;
    808                if (imax == 0) {
    809                        return 0;
    810                }
    811
    812                return get1DIndexFromShape(pos);
    813        }
    814
    815        /**
    816         * @since 2.0
    817         */
    818        protected int getFirst1DIndex() {
    819                if (shape == null) {
    820                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    821                }
    822                return stride == null ? 0 : offset;
    823        }
    824
    825        @Override
    826        public int get1DIndex(final int... n) {
    827                if (n.length == 0 && shape.length == 0)
    828                        return offset;
    829
    830                return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n);
    831        }
    832
    833        private static void throwAIOOBException(int i, int s, int d) {
    834                throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s
    835                                + "] in dimension " + d);
    836        }
    837
    838        /**
    839         * @param i
    840         * @return the index on the data array corresponding to that location
    841         */
    842        protected int get1DIndex(int i) {
    843                if (shape == null) {
    844                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    845                }
    846                if (shape.length > 1) {
    847                        logger.error("This dataset is not 1D but was addressed as such");
    848                        throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such");
    849                }
    850                if (i < 0) {
    851                        i += shape[0];
    852                }
    853                if (i < 0 || i >= shape[0]) {
    854                        throwAIOOBException(i, shape[0], 0);
    855                }
    856                return stride == null ? i : i*stride[0] + offset;
    857        }
    858
    859        /**
    860         * @param i
    861         * @param j
    862         * @return the index on the data array corresponding to that location
    863         */
    864        protected int get1DIndex(int i, int j) {
    865                if (shape == null) {
    866                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    867                }
    868                if (shape.length != 2) {
    869                        logger.error("This dataset is not 2D but was addressed as such");
    870                        throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such");
    871                }
    872                if (i < 0) {
    873                        i += shape[0];
    874                }
    875                if (i < 0 || i >= shape[0]) {
    876                        throwAIOOBException(i, shape[0], 0);
    877                }
    878                if (j < 0) {
    879                        j += shape[1];
    880                }
    881                if (j < 0 || j >= shape[1]) {
    882                        throwAIOOBException(i, shape[1], 1);
    883                }
    884                return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset;
    885        }
    886
    887        protected int get1DIndexFromShape(final int[] n) {
    888                return get1DIndexFromShape(shape, n);
    889        }
    890
    891        protected static int get1DIndexFromShape(final int[] shape, final int[] n) {
    892                if (shape == null) {
    893                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    894                }
    895                final int rank = shape.length;
    896                if (rank != n.length) {
    897                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
    898                        logger.error(errMsg);
    899                        throw new IllegalArgumentException(errMsg);
    900                }
    901                int index = 0;
    902                for (int i = 0; i < rank; i++) {
    903                        final int si = shape[i];
    904                        int ni = n[i];
    905                        if (ni < 0) {
    906                                ni += si;
    907                        }
    908                        if (ni < 0 || ni >= si) {
    909                                throwAIOOBException(ni, si, i);
    910                        }
    911                        index = index * si + ni;
    912                }
    913
    914                return index;
    915        }
    916
    917        private int get1DIndexFromStrides(final int[] n) {
    918                return get1DIndexFromStrides(shape, stride, offset, n);
    919        }
    920
    921        private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) {
    922                if (shape == null) {
    923                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    924                }
    925                final int rank = shape.length;
    926                if (rank != n.length) {
    927                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
    928                        logger.error(errMsg);
    929                        throw new IllegalArgumentException(errMsg);
    930                }
    931                int index = offset;
    932                for (int i = 0; i < rank; i++) {
    933                        final int st = stride[i];
    934                        if (st != 0) { // not broadcasted
    935                                final int si = shape[i];
    936                                int ni = n[i];
    937                                if (ni < 0) {
    938                                        ni += si;
    939                                }
    940                                if (ni < 0 || ni >= si) {
    941                                        throwAIOOBException(ni, si, i);
    942                                }
    943                                index += st * ni;
    944                        }
    945                }
    946                return index;
    947        }
    948
    949        @Override
    950        public int[] getNDPosition(final int n) {
    951                if (isIndexInRange(n)) {
    952                        throw new IllegalArgumentException("Index provided " + n
    953                                        + "is larger then the size of the containing array");
    954                }
    955
    956                return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n);
    957        }
    958
    959        private boolean isIndexInRange(final int n) {
    960                if (stride == null) {
    961                        return n >= size;
    962                }
    963                return n >= getBufferLength();
    964        }
    965
    966        /**
    967         * @return entire buffer length
    968         */
    969        abstract protected int getBufferLength();
    970
    971        private int[] getNDPositionFromStrides(int n) {
    972                n -= offset;
    973                int rank = shape.length;
    974                if (rank == 1) {
    975                        return new int[] { n / stride[0] };
    976                }
    977
    978                int[] output = new int[rank];
    979                int i = 0;
    980                while (i != n) { // TODO find more efficient way than this exhaustive search
    981                        int j = rank - 1;
    982                        for (; j >= 0; j--) {
    983                                output[j]++;
    984                                i += stride[j];
    985                                if (output[j] >= shape[j]) {
    986                                        output[j] = 0;
    987                                        i -= shape[j] * stride[j];
    988                                } else {
    989                                        break;
    990                                }
    991                        }
    992                        if (j == -1) {
    993                                logger.error("Index was not found in this strided dataset");
    994                                throw new IllegalArgumentException("Index was not found in this strided dataset");
    995                        }
    996                }
    997
    998                return output;
    999        }
    1000
    1001        @Override
    1002        public int checkAxis(int axis) {
    1003                return ShapeUtils.checkAxis(shape.length, axis);
    1004        }
    1005
    1006        @Deprecated
    1007        protected static int checkAxis(int rank, int axis) {
    1008                return ShapeUtils.checkAxis(rank, axis);
    1009        }
    1010
    1011        protected static final char BLOCK_OPEN = '[';
    1012        protected static final char BLOCK_CLOSE = ']';
    1013
    1014        @Override
    1015        public String toString() {
    1016                final int rank = shape == null ? 0 : shape.length;
    1017                final StringBuilder out = new StringBuilder();
    1018
    1019                if (DTypeUtils.isDTypeElemental(getDType())) {
    1020                        out.append("Dataset ");
    1021                } else {
    1022                        out.append("Compound dataset (");
    1023                        out.append(getElementsPerItem());
    1024                        out.append(") ");
    1025                }
    1026
    1027                if (name != null && name.length() > 0) {
    1028                        out.append("'");
    1029                        out.append(name);
    1030                        out.append("' has shape ");
    1031                } else {
    1032                        out.append("shape is ");
    1033                }
    1034
    1035                out.append(BLOCK_OPEN);
    1036                if (rank > 0) {
    1037                        out.append(shape[0]);
    1038                }
    1039                for (int i = 1; i < rank; i++) {
    1040                        out.append(", " + shape[i]);
    1041                }
    1042                out.append(BLOCK_CLOSE);
    1043                return out.toString();
    1044        }
    1045
    1046        @Override
    1047        public String toString(boolean showData) {
    1048                if (!showData) {
    1049                        return toString();
    1050                }
    1051
    1052                if (size == 0) {
    1053                        return "[]";
    1054                }
    1055
    1056                final int rank = shape == null ? 0 : shape.length;
    1057                final StringBuilder out = new StringBuilder();
    1058
    1059                if (rank > 0) {
    1060                        int[] pos = new int[rank];
    1061                        final StringBuilder lead = new StringBuilder();
    1062                        printBlocks(out, lead, 0, pos);
    1063                } else {
    1064                        out.append(getString());
    1065                }
    1066                return out.toString();
    1067        }
    1068
    1069        /**
    1070         * Limit to strings output via the toString() method
    1071         */
    1072        private static int maxStringLength = 120;
    1073
    1074        /**
    1075         * Set maximum line length for toString() method
    1076         * @param maxLineLength
    1077         */
    1078        public static void setMaxLineLength(int maxLineLength) {
    1079                maxStringLength = maxLineLength;
    1080        }
    1081
    1082        /**
    1083         * @return maximum line length for toString() method
    1084         */
    1085        public static int getMaxLineLength() {
    1086                return maxStringLength;
    1087        }
    1088
    1089        /**
    1090         * Limit to number of sub-blocks output via the toString() method
    1091         */
    1092        private static final int MAX_SUBBLOCKS = 6;
    1093
    1094        private final static String SEPARATOR = ",";
    1095        private final static String SPACE = " ";
    1096        private final static String ELLIPSIS = "...";
    1097        private final static String NEWLINE = "\n";
    1098
    1099        /**
    1100         * Make a line of output for last dimension of dataset
    1101         *
    1102         * @param start
    1103         * @return line
    1104         */
    1105        private StringBuilder makeLine(final int end, final int[] start) {
    1106                StringBuilder line = new StringBuilder();
    1107                final int[] pos;
    1108                if (end >= start.length) {
    1109                        pos = Arrays.copyOf(start, end + 1);
    1110                } else {
    1111                        pos = start;
    1112                }
    1113                pos[end] = 0;
    1114                line.append(BLOCK_OPEN);
    1115                line.append(getString(pos));
    1116
    1117                final int length = shape[end];
    1118
    1119                // trim elements printed if length exceed estimate of maximum elements
    1120                int excess = length - maxStringLength / 3; // space + number + separator
    1121                int midIndex = -1;
    1122                if (excess > 0) {
    1123                        int index = (length - excess) / 2;
    1124                        for (int y = 1; y < index; y++) {
    1125                                line.append(SEPARATOR + SPACE);
    1126                                pos[end] = y;
    1127                                line.append(getString(pos));
    1128                        }
    1129                        midIndex = line.length() + 2;
    1130                        index = (length + excess) / 2;
    1131                        for (int y = index; y < length; y++) {
    1132                                line.append(SEPARATOR + SPACE);
    1133                                pos[end] = y;
    1134                                line.append(getString(pos));
    1135                        }
    1136                } else {
    1137                        for (int y = 1; y < length; y++) {
    1138                                line.append(SEPARATOR + SPACE);
    1139                                pos[end] = y;
    1140                                line.append(getString(pos));
    1141                        }
    1142                }
    1143                line.append(BLOCK_CLOSE);
    1144
    1145                // trim string down to limit
    1146                int lineLength = line.length();
    1147                excess = lineLength - maxStringLength - ELLIPSIS.length() - 1;
    1148                if (excess > 0) {
    1149                        int index = (lineLength - excess) / 2;
    1150                        if (midIndex > 0 && index > midIndex) {
    1151                                index = midIndex;
    1152                        } else {
    1153                                index = line.lastIndexOf(SEPARATOR, index) + 2;
    1154                        }
    1155                        StringBuilder out = new StringBuilder(line.subSequence(0, index));
    1156                        out.append(ELLIPSIS + SEPARATOR);
    1157                        index = (lineLength + excess) / 2;
    1158                        if (midIndex > 0 && index <= midIndex) {
    1159                                index = midIndex - 1;
    1160                        } else {
    1161                                index = line.indexOf(SEPARATOR, index) + 1;
    1162                        }
    1163                        out.append(line.subSequence(index, lineLength));
    1164                        return out;
    1165                } else if (midIndex > 0) { // add ellipsis
    1166                        StringBuilder out = new StringBuilder(line.subSequence(0, midIndex));
    1167                        out.append(ELLIPSIS + SEPARATOR + SPACE);
    1168                        out.append(line.subSequence(midIndex, lineLength));
    1169                        return out;
    1170                }
    1171
    1172                return line;
    1173        }
    1174
    1175        /**
    1176         * recursive method to print blocks
    1177         */
    1178        private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) {
    1179                if (out.length() > 0) {
    1180                        char last = out.charAt(out.length() - 1);
    1181                        if (last != BLOCK_OPEN) {
    1182                                out.append(lead);
    1183                        }
    1184                }
    1185                final int end = getRank() - 1;
    1186                if (level != end) {
    1187                        out.append(BLOCK_OPEN);
    1188                        int length = shape[level];
    1189
    1190                        // first sub-block
    1191                        pos[level] = 0;
    1192                        StringBuilder newlead = new StringBuilder(lead);
    1193                        newlead.append(SPACE);
    1194                        printBlocks(out, newlead, level + 1, pos);
    1195                        if (length < 2) { // escape
    1196                                out.append(BLOCK_CLOSE);
    1197                                return;
    1198                        }
    1199
    1200                        out.append(SEPARATOR + NEWLINE);
    1201                        for (int i = level + 1; i < end; i++) {
    1202                                out.append(NEWLINE);
    1203                        }
    1204
    1205                        // middle sub-blocks
    1206                        if (length < MAX_SUBBLOCKS) {
    1207                                for (int x = 1; x < length - 1; x++) {
    1208                                        pos[level] = x;
    1209                                        printBlocks(out, newlead, level + 1, pos);
    1210                                        if (end <= level + 1) {
    1211                                                out.append(SEPARATOR + NEWLINE);
    1212                                        } else {
    1213                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
    1214                                        }
    1215                                }
    1216                        } else {
    1217                                final int excess = length - MAX_SUBBLOCKS;
    1218                                int xmax = (length - excess) / 2;
    1219                                for (int x = 1; x < xmax; x++) {
    1220                                        pos[level] = x;
    1221                                        printBlocks(out, newlead, level + 1, pos);
    1222                                        if (end <= level + 1) {
    1223                                                out.append(SEPARATOR + NEWLINE);
    1224                                        } else {
    1225                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
    1226                                        }
    1227                                }
    1228                                out.append(newlead);
    1229                                out.append(ELLIPSIS + SEPARATOR + NEWLINE);
    1230                                xmax = (length + excess) / 2;
    1231                                for (int x = xmax; x < length - 1; x++) {
    1232                                        pos[level] = x;
    1233                                        printBlocks(out, newlead, level + 1, pos);
    1234                                        if (end <= level + 1) {
    1235                                                out.append(SEPARATOR + NEWLINE);
    1236                                        } else {
    1237                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
    1238                                        }
    1239                                }
    1240                        }
    1241
    1242                        // last sub-block
    1243                        pos[level] = length - 1;
    1244                        printBlocks(out, newlead, level + 1, pos);
    1245                        out.append(BLOCK_CLOSE);
    1246                } else {
    1247                        out.append(makeLine(end, pos));
    1248                }
    1249        }
    1250
    1251        @Override
    1252        public Dataset squeezeEnds() {
    1253                return squeeze(true);
    1254        }
    1255
    1256        @Override
    1257        public Dataset squeeze() {
    1258                return squeeze(false);
    1259        }
    1260
    1261        @Override
    1262        public Dataset squeeze(boolean onlyFromEnds) {
    1263                final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds);
    1264                final int[] oshape = shape;
    1265                if (stride == null) {
    1266                        shape = tshape;
    1267                } else {
    1268                        int rank = shape.length;
    1269                        int trank = tshape.length;
    1270                        if (trank < rank) {
    1271                                int[] tstride = new int[tshape.length];
    1272                                if (onlyFromEnds) {
    1273                                        for (int i = 0; i < rank; i++) {
    1274                                                if (shape[i] != 1) {
    1275                                                        for (int k = 0; k < trank; k++) {
    1276                                                                tstride[k] = stride[i++];
    1277                                                        }
    1278                                                        break;
    1279                                                }
    1280                                        }
    1281                                } else {
    1282                                        int t = 0;
    1283                                        for (int i = 0; i < rank; i++) {
    1284                                                if (shape[i] != 1) {
    1285                                                        tstride[t++] = stride[i];
    1286                                                }
    1287                                        }
    1288                                }
    1289                                shape = tshape;
    1290                                stride = tstride;
    1291                        }
    1292                }
    1293
    1294                setDirty();
    1295                reshapeMetadata(oshape, shape);
    1296                return this;
    1297        }
    1298
    1299        @Override
    1300        public boolean isCompatibleWith(final ILazyDataset g) {
    1301                return ShapeUtils.areShapesCompatible(shape, g.getShape());
    1302        }
    1303
    1304        @Override
    1305        public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException {
    1306                ShapeUtils.checkCompatibility(this, g);
    1307        }
    1308
    1309        @Override
    1310        public Dataset reshape(final int... shape) {
    1311                Dataset a;
    1312                try {
    1313                        a = getView(true);
    1314                        a.setShape(shape);
    1315                } catch (IllegalArgumentException e) {
    1316                        a = clone();
    1317                        a.setShape(shape);
    1318                }
    1319                return a;
    1320        }
    1321
    1322        /**
    1323         * @param start
    1324         * @param stop
    1325         * @param step
    1326         * @return number of steps to take
    1327         */
    1328        protected static int calcSteps(final double start, final double stop, final double step) {
    1329                return Math.max(0, (int) Math.ceil((stop - start) / step));
    1330        }
    1331
    1332        @Override
    1333        public boolean isComplex() {
    1334                int type = getDType();
    1335                return type == COMPLEX64 || type == COMPLEX128;
    1336        }
    1337
    1338        @Override
    1339        public Dataset getRealPart() {
    1340                return this;
    1341        }
    1342
    1343        @Override
    1344        public Dataset getRealView() {
    1345                return getView(true);
    1346        }
    1347
    1348        @Override
    1349        public Dataset getSlice(final int[] start, final int[] stop, final int[] step) {
    1350                return getSlice(new SliceND(shape, start, stop, step));
    1351        }
    1352
    1353        @Override
    1354        public Dataset getSlice(Slice... slice) {
    1355                return getSlice(new SliceND(shape, slice));
    1356        }
    1357
    1358        @Override
    1359        public Dataset getSlice(IMonitor monitor, Slice... slice) {
    1360                return getSlice(slice);
    1361        }
    1362
    1363        @Override
    1364        public Dataset getSlice(IMonitor monitor, SliceND slice) {
    1365                return getSlice(slice);
    1366        }
    1367
    1368        @Override
    1369        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) {
    1370                return getSlice(start, stop, step);
    1371        }
    1372
    1373        /**
    1374         * Get a slice of the dataset. The returned dataset is a copied selection of items
    1375         * @param slice
    1376         * @return The dataset of the sliced data
    1377         */
    1378        @Override
    1379        public Dataset getSlice(final SliceND slice) {
    1380                SliceIterator it = (SliceIterator) getSliceIterator(slice);
    1381                AbstractDataset s = getSlice(it);
    1382                s.metadata = copyMetadata();
    1383                s.setDirty();
    1384                s.sliceMetadata(true, slice);
    1385                return s;
    1386        }
    1387
    1388        /**
    1389         * Get a slice of the dataset. The returned dataset is a copied selection of items
    1390         *
    1391         * @param iterator Slice iterator
    1392         * @return The dataset of the sliced data
    1393         */
    1394        abstract public AbstractDataset getSlice(final SliceIterator iterator);
    1395
    1396        @SuppressWarnings("deprecation")
    1397        @Override
    1398        public Dataset setSlice(final Object obj, final SliceND slice) {
    1399                Dataset ds;
    1400                if (obj instanceof Dataset) {
    1401                        ds = (Dataset) obj;
    1402                } else if (obj instanceof IDataset) {
    1403                        ds = DatasetUtils.convertToDataset((IDataset) obj);
    1404                } else {
    1405                        int dtype = getDType();
    1406                        if (dtype != Dataset.BOOL) {
    1407                                dtype = DTypeUtils.getLargestDType(dtype);
    1408                        }
    1409                        ds = DatasetFactory.createFromObject(getElementsPerItem(), dtype, obj);
    1410                }
    1411
    1412                return setSlicedView(getSliceView(slice), ds);
    1413        }
    1414
    1415        @Override
    1416        public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) {
    1417                return setSlice(obj, new SliceND(shape, start, stop, step));
    1418        }
    1419
    1420        /**
    1421         * Set a view of current dataset to given dataset with broadcasting
    1422         * @param view
    1423         * @param d
    1424         * @return this dataset
    1425         */
    1426        abstract Dataset setSlicedView(Dataset view, Dataset d);
    1427
    1428        @Override
    1429        public Dataset setSlice(Object obj, Slice... slice) {
    1430                if (slice == null || slice.length == 0) {
    1431                        return setSlice(obj, new SliceND(shape));
    1432                }
    1433                return setSlice(obj, new SliceND(shape, slice));
    1434        }
    1435
    1436        @Override
    1437        public boolean all() {
    1438                return Comparisons.allTrue(this);
    1439        }
    1440
    1441        @Override
    1442        public BooleanDataset all(final int axis) {
    1443                return Comparisons.allTrue(this, axis);
    1444        }
    1445
    1446        @Override
    1447        public boolean any() {
    1448                return Comparisons.anyTrue(this);
    1449        }
    1450
    1451        @Override
    1452        public BooleanDataset any(final int axis) {
    1453                return Comparisons.anyTrue(this, axis);
    1454        }
    1455
    1456        @Override
    1457        public Dataset ifloorDivide(final Object o) {
    1458                return idivide(o).ifloor();
    1459        }
    1460
    1461        @Override
    1462        public double residual(final Object o) {
    1463                return residual(o, null, false);
    1464        }
    1465
    1466        @Override
    1467        public double residual(final Object o, boolean ignoreNaNs) {
    1468                return residual(o, null, ignoreNaNs);
    1469        }
    1470
    1471        /**
    1472         * @since 2.0
    1473         */
    1474        @SuppressWarnings("unchecked")
    1475        protected StatisticsMetadata<Number> getStats() {
    1476                StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class);
    1477                if (md == null || md.isDirty(this)) {
    1478                        md = new StatisticsMetadataImpl<Number>();
    1479                        md.initialize(this);
    1480                        setMetadata(md);
    1481                }
    1482                return md;
    1483        }
    1484
    1485        /**
    1486         * @since 2.0
    1487         */
    1488        @SuppressWarnings("unchecked")
    1489        protected StatisticsMetadata<String> getStringStats() {
    1490                StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class);
    1491                if (md == null || md.isDirty(this)) {
    1492                        md = new StatisticsMetadataImpl<String>();
    1493                        md.initialize(this);
    1494                        setMetadata(md);
    1495                }
    1496                return md;
    1497        }
    1498
    1499        @Override
    1500        public Number max(boolean... ignoreInvalids) {
    1501                return getStats().getMaximum(ignoreInvalids);
    1502        }
    1503
    1504        @Override
    1505        public Dataset max(int axis, boolean... ignoreInvalids) {
    1506                return getStats().getMaximum(axis, ignoreInvalids);
    1507        }
    1508
    1509        @Override
    1510        public Dataset max(int[] axes, boolean... ignoreInvalids) {
    1511                return getStats().getMaximum(axes, ignoreInvalids);
    1512        }
    1513
    1514        @Override
    1515        public Number min(boolean... ignoreInvalids) {
    1516                return getStats().getMinimum(ignoreInvalids);
    1517        }
    1518
    1519        @Override
    1520        public Dataset min(int axis, boolean... ignoreInvalids) {
    1521                return getStats().getMinimum(axis, ignoreInvalids);
    1522        }
    1523
    1524        @Override
    1525        public Dataset min(int[] axes, boolean... ignoreInvalids) {
    1526                return getStats().getMinimum(axes, ignoreInvalids);
    1527        }
    1528
    1529        @Override
    1530        public int argMax(boolean... ignoreInvalids) {
    1531                return getFlat1DIndex(maxPos(ignoreInvalids));
    1532        }
    1533
    1534        /**
    1535         * @since 2.0
    1536         */
    1537        @Override
    1538        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
    1539                return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids);
    1540        }
    1541
    1542        @Override
    1543        public int argMin(boolean... ignoreInvalids) {
    1544                return getFlat1DIndex(minPos(ignoreInvalids));
    1545        }
    1546
    1547        /**
    1548         * @since 2.0
    1549         */
    1550        @Override
    1551        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
    1552                return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids);
    1553        }
    1554
    1555        @Override
    1556        public Number peakToPeak(boolean... ignoreInvalids) {
    1557                return DTypeUtils.fromDoubleToBiggestNumber(max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue(), getDType());
    1558        }
    1559
    1560        @Override
    1561        public Dataset peakToPeak(int axis,  boolean... ignoreInvalids) {
    1562                return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids));
    1563        }
    1564
    1565        @Override
    1566        public Dataset peakToPeak(int[] axes,  boolean... ignoreInvalids) {
    1567                return Maths.subtract(max(axes, ignoreInvalids), min(axes, ignoreInvalids));
    1568        }
    1569
    1570        @Override
    1571        public long count(boolean... ignoreInvalids) {
    1572                return getStats().getCount(ignoreInvalids);
    1573        }
    1574
    1575        @Override
    1576        public Dataset count(int axis, boolean... ignoreInvalids) {
    1577                return getStats().getCount(axis, ignoreInvalids);
    1578        }
    1579
    1580        @Override
    1581        public Dataset count(int[] axes, boolean... ignoreInvalids) {
    1582                return getStats().getCount(axes, ignoreInvalids);
    1583        }
    1584
    1585        @Override
    1586        public Object sum(boolean... ignoreInvalids) {
    1587                return getStats().getSum(ignoreInvalids);
    1588        }
    1589
    1590        @Override
    1591        public Dataset sum(int axis, boolean... ignoreInvalids) {
    1592                return getStats().getSum(axis, ignoreInvalids);
    1593        }
    1594
    1595        @Override
    1596        public Dataset sum(int[] axes, boolean... ignoreInvalids) {
    1597                return getStats().getSum(axes, ignoreInvalids);
    1598        }
    1599
    1600        @Override
    1601        public Object product(boolean... ignoreInvalids) {
    1602                return Stats.product(this, ignoreInvalids);
    1603        }
    1604
    1605        @Override
    1606        public Dataset product(int axis, boolean... ignoreInvalids) {
    1607                return Stats.product(this, axis, ignoreInvalids);
    1608        }
    1609
    1610        @Override
    1611        public Dataset product(int[] axes, boolean... ignoreInvalids) {
    1612                return Stats.product(this, axes, ignoreInvalids);
    1613        }
    1614
    1615        @Override
    1616        public Object mean(boolean... ignoreInvalids) {
    1617                return getStats().getMean(ignoreInvalids);
    1618        }
    1619
    1620        @Override
    1621        public Dataset mean(int axis, boolean... ignoreInvalids) {
    1622                return getStats().getMean(axis, ignoreInvalids);
    1623        }
    1624
    1625        @Override
    1626        public Dataset mean(int[] axes, boolean... ignoreInvalids) {
    1627                return getStats().getMean(axes, ignoreInvalids);
    1628        }
    1629
    1630        @Override
    1631        public double variance() {
    1632                return variance(false);
    1633        }
    1634
    1635        @Override
    1636        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
    1637                return getStats().getVariance(isWholePopulation, ignoreInvalids);
    1638        }
    1639
    1640        @Override
    1641        public Dataset variance(int axis) {
    1642                return getStats().getVariance(axis, false);
    1643        }
    1644
    1645        @Override
    1646        public Dataset variance(int[] axes) {
    1647                return getStats().getVariance(axes, false);
    1648        }
    1649
    1650        @Override
    1651        public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    1652                return getStats().getVariance(axis, isWholePopulation, ignoreInvalids);
    1653        }
    1654
    1655        @Override
    1656        public Dataset variance(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) {
    1657                return getStats().getVariance(axes, isWholePopulation, ignoreInvalids);
    1658        }
    1659
    1660        @Override
    1661        public double stdDeviation() {
    1662                return Math.sqrt(variance());
    1663        }
    1664
    1665        @Override
    1666        public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) {
    1667                return Math.sqrt(variance(isWholePopulation, ignoreInvalids));
    1668        }
    1669
    1670        @Override
    1671        public Dataset stdDeviation(int axis) {
    1672                return Maths.sqrt(variance(axis, false));
    1673        }
    1674
    1675        @Override
    1676        public Dataset stdDeviation(int[] axes) {
    1677                return Maths.sqrt(variance(axes, false));
    1678        }
    1679
    1680        @Override
    1681        public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    1682                return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids));
    1683        }
    1684
    1685        @Override
    1686        public Dataset stdDeviation(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) {
    1687                return Maths.sqrt(variance(axes, isWholePopulation, ignoreInvalids));
    1688        }
    1689
    1690        @Override
    1691        public double rootMeanSquare(boolean... ignoreInvalids) {
    1692                StatisticsMetadata<Number> stats = getStats();
    1693                final double mean = stats.getMean(ignoreInvalids).doubleValue();
    1694                final double var = stats.getVariance(true, ignoreInvalids);
    1695                return Math.sqrt(var + mean * mean);
    1696        }
    1697
    1698        @Override
    1699        public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
    1700                StatisticsMetadata<Number> stats = getStats();
    1701                Dataset v = stats.getVariance(axis, true, ignoreInvalids);
    1702                Dataset m = stats.getMean(axis, ignoreInvalids);
    1703                Dataset result = Maths.multiply(m, m);
    1704                return Maths.sqrt(result.iadd(v));
    1705        }
    1706
    1707        @Override
    1708        public Dataset rootMeanSquare(int[] axes, boolean... ignoreInvalids) {
    1709                StatisticsMetadata<Number> stats = getStats();
    1710                Dataset v = stats.getVariance(axes, true, ignoreInvalids);
    1711                Dataset m = stats.getMean(axes, ignoreInvalids);
    1712                Dataset result = Maths.multiply(m, m);
    1713                return Maths.sqrt(result.iadd(v));
    1714        }
    1715
    1716        /**
    1717         * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards.
    1718         *
    1719         * @param dindex
    1720         * @param sindex
    1721         * @param src
    1722         *            is the source data buffer
    1723         */
    1724        protected abstract void setItemDirect(final int dindex, final int sindex, final Object src);
    1725
    1726        /**
    1727         * @return error broadcasted to current shape
    1728         */
    1729        private Dataset getBroadcastedInternalError() {
    1730                if (shape == null) {
    1731                        throw new IllegalArgumentException("Cannot get error for null dataset");
    1732                }
    1733                ILazyDataset led = super.getErrors();
    1734                if (led == null)
    1735                        return null;
    1736
    1737                Dataset ed = null;
    1738                try {
    1739                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
    1740                } catch (DatasetException e) {
    1741                        logger.error("Could not get data from lazy dataset", e);
    1742                }
    1743                if (led != ed) {
    1744                        setErrors(ed); // set back
    1745                }
    1746
    1747                return ed.getBroadcastView(shape);
    1748        }
    1749
    1750        @Override
    1751        public Dataset getErrors() {
    1752                Dataset ed = getBroadcastedInternalError();
    1753                if (ed == null)
    1754                        return null;
    1755
    1756                return ed;
    1757        }
    1758
    1759        @Override
    1760        public double getError() {
    1761                Dataset ed = getBroadcastedInternalError();
    1762                if (ed == null)
    1763                        return 0;
    1764
    1765                return ed.getDouble();
    1766        }
    1767
    1768        @Override
    1769        public double getError(final int i) {
    1770                Dataset ed = getBroadcastedInternalError();
    1771                if (ed == null)
    1772                        return 0;
    1773
    1774                return ed.getDouble(i);
    1775        }
    1776
    1777        @Override
    1778        public double getError(final int i, final int j) {
    1779                Dataset ed = getBroadcastedInternalError();
    1780                if (ed == null)
    1781                        return 0;
    1782
    1783                return ed.getDouble(i, j);
    1784        }
    1785
    1786        @Override
    1787        public double getError(int... pos) {
    1788                Dataset ed = getBroadcastedInternalError();
    1789                if (ed == null)
    1790                        return 0;
    1791
    1792                return ed.getDouble(pos);
    1793        }
    1794
    1795        @Override
    1796        public double[] getErrorArray(final int i) {
    1797                Dataset ed = getBroadcastedInternalError();
    1798                if (ed == null)
    1799                        return null;
    1800
    1801                return new double[] {getError(i)};
    1802        }
    1803
    1804        @Override
    1805        public double[] getErrorArray(final int i, final int j) {
    1806                Dataset ed = getBroadcastedInternalError();
    1807                if (ed == null)
    1808                        return null;
    1809
    1810                return new double[] {getError(i, j)};
    1811        }
    1812
    1813        @Override
    1814        public double[] getErrorArray(int... pos) {
    1815                Dataset ed = getBroadcastedInternalError();
    1816                if (ed == null)
    1817                        return null;
    1818
    1819                return new double[] {getError(pos)};
    1820        }
    1821
    1822        protected Dataset getInternalSquaredError() {
    1823                Dataset sed = getErrorBuffer().getBroadcastView(shape);
    1824                return sed;
    1825        }
    1826
    1827        @Override
    1828        public Dataset getErrorBuffer() {
    1829                ErrorMetadata emd = getErrorMetadata();
    1830                if (emd == null)
    1831                        return null;
    1832
    1833                if (!(emd instanceof ErrorMetadataImpl)) {
    1834                        ILazyDataset led = emd.getError();
    1835                        Dataset ed;
    1836                        try {
    1837                                ed = DatasetUtils.sliceAndConvertLazyDataset(led);
    1838                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1839                                setMetadata(emd);
    1840                                emd.setError(ed);
    1841                        } catch (MetadataException me) {
    1842                                logger.error("Could not create metadata", me);
    1843                        } catch (DatasetException e) {
    1844                                logger.error("Could not get data from lazy dataset", e);
    1845                        }
    1846                }
    1847
    1848                return ((ErrorMetadataImpl) emd).getSquaredError();
    1849        }
    1850
    1851        /**
    1852         * Set a copy of the buffer that backs the (squared) error data
    1853         * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset
    1854         */
    1855        @Override
    1856        public void setErrorBuffer(Serializable buffer) {
    1857                if (shape == null) {
    1858                        throw new IllegalArgumentException("Cannot set error buffer for null dataset");
    1859                }
    1860                if (buffer == null) {
    1861                        clearMetadata(ErrorMetadata.class);
    1862                        return;
    1863                }
    1864
    1865                IDataset d = (IDataset) createFromSerializable(buffer, false);
    1866                ErrorMetadata emd = getErrorMetadata();
    1867                if (!(emd instanceof ErrorMetadataImpl)) {
    1868                        try {
    1869                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1870                                setMetadata(emd);
    1871                        } catch (MetadataException me) {
    1872                                logger.error("Could not create metadata", me);
    1873                        }
    1874                }
    1875                ((ErrorMetadataImpl) emd).setSquaredError(d);
    1876        }
    1877}