    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.Serializable;
    016import java.lang.reflect.Array;
    017import java.text.Format;
    018import java.util.Arrays;
    019import java.util.List;
    020
    021import org.eclipse.january.DatasetException;
    022import org.eclipse.january.IMonitor;
    023import org.eclipse.january.MetadataException;
    024import org.eclipse.january.metadata.ErrorMetadata;
    025import org.eclipse.january.metadata.MetadataFactory;
    026import org.eclipse.january.metadata.StatisticsMetadata;
    027import org.eclipse.january.metadata.internal.ErrorMetadataImpl;
    028import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
    029
    030/**
    031 * Generic container class for data
    032 * <p>
    033 * Each subclass has an array of primitive types, elements of this array are grouped or
    034 * compounded to make items
    035 * <p>
    036 * Data items can be boolean, integer, float, complex float, vector float, etc
    037 */
    038public abstract class AbstractDataset extends LazyDatasetBase implements Dataset {
    039        // pin UID to base class
    040        private static final long serialVersionUID = Dataset.serialVersionUID;
    041
    042        protected int size; // number of items
    043
    044        transient protected AbstractDataset base; // is null when not a view
    045        protected int[] stride; // can be null for row-major, contiguous datasets
    046        protected int offset;
    047
    048        /**
    049         * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as
    050         * wanted
    051         */
    052        protected Serializable odata = null;
    053
    054        /**
    055         * Set aliased data as base data
    056         */
    057        abstract protected void setData();
    058
    059        /**
    060         * Constructor required for serialisation.
    061         */
    062        public AbstractDataset() {
    063        }
    064
    065        @Override
    066        public synchronized Dataset synchronizedCopy() {
    067                return clone();
    068        }
    069
    070        @Override
    071        public int hashCode() {
    072                return getStats().getHash(shape);
    073        }
    074
    075        @Override
    076        abstract public AbstractDataset clone();
    077
    078        protected Format stringFormat = null;
    079
    080        @Override
    081        public void setStringFormat(Format format) {
    082                stringFormat = format;
    083        }
    084
    085        @Override
    086        public Dataset copy(final int dtype) {
    087                return copy(DTypeUtils.getInterface(dtype));
    088        }
    089
    090        @Override
    091        public <T extends Dataset> T copy(Class<T> clazz) {
    092                return DatasetUtils.copy(clazz, this);
    093        }
    094
    095        @Override
    096        public Dataset cast(final int dtype) {
    097                return cast(DTypeUtils.getInterface(dtype));
    098        }
    099
    100        @SuppressWarnings("unchecked")
    101        @Override
    102        public <T extends Dataset> T cast(Class<T> clazz) {
    103                if (getClass().equals(clazz)) {
    104                        return (T) this;
    105                }
    106                return DatasetUtils.cast(clazz, this);
    107        }
    108
    109        /**
    110         * {@inheritDoc Dataset#cast(int, Class, boolean)}
    111         * @since 2.3
    112         */
    113        @SuppressWarnings("unchecked")
    114        @Override
    115        public <T extends Dataset> T cast(final int isize, Class<T> clazz, final boolean repeat) {
    116                if (getClass().equals(clazz) && getElementsPerItem() == isize) {
    117                        return (T) this;
    118                }
    119                return DatasetUtils.cast(isize, clazz, this, repeat);
    120        }
    121
    122        @Override
    123        public Dataset cast(final boolean repeat, final int dtype, final int isize) {
    124                return cast(isize, DTypeUtils.getInterface(dtype), repeat);
    125        }
    126
    127        @Override
    128        abstract public AbstractDataset getView(boolean deepCopyMetadata);
    129
    130        /**
    131         * Copy fields from original to view
    132         * @param orig
    133         * @param view
    134         * @param clone if true, then clone everything but bulk data
    135         * @param cloneMetadata if true, clone metadata
    136         */
    137        protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) {
    138                view.name = orig.getName();
    139                view.size = orig.getSize();
    140                view.odata = orig.getBuffer();
    141                view.offset = orig.getOffset();
    142                view.base = orig instanceof AbstractDataset ? ((AbstractDataset) orig).base : null;
    143
    144                if (clone) {
    145                        view.shape = orig.getShapeRef() == null ? null : orig.getShape();
    146                        view.stride = orig instanceof AbstractDataset && ((AbstractDataset) orig).stride != null ?
    147                                        ((AbstractDataset) orig).stride.clone() : null;
    148                } else {
    149                        view.shape = orig.getShapeRef();
    150                        view.stride = orig instanceof AbstractDataset ? ((AbstractDataset) orig).stride : null;
    151                }
    152
    153                view.metadata = getMetadataMap(orig, cloneMetadata);
    154                int odtype = orig.getDType();
    155                int vdtype = view.getDType();
    156                if (odtype != vdtype) {
    157                        view.setDirty();
    158                }
    159        }
    160
    161        @Override
    162        public IntegerDataset getIndices() {
    163                final IntegerDataset ret = DatasetUtils.indices(shape);
    164                if (getName() != null) {
    165                        ret.setName("Indices of " + getName());
    166                }
    167                return ret;
    168        }
    169
    170        @Override
    171        public Dataset getTransposedView(int... axes) {
    172                axes = checkPermutatedAxes(shape, axes);
    173
    174                AbstractDataset t = getView(true);
    175                if (axes == null || getRank() == 1)
    176                        return t;
    177
    178                int rank = shape.length;
    179                int[] tstride = new int[rank];
    180                int[] toffset = new int[1];
    181                int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset);
    182                int[] nstride = new int[rank];
    183                for (int i = 0; i < rank; i++) {
    184                        final int ax = axes[i];
    185                        nstride[i] = tstride[ax];
    186                        nshape[i] = shape[ax];
    187                }
    188                t.shape = nshape;
    189                t.stride = nstride;
    190                t.offset = toffset[0];
    191                t.base = this;
    192                t.setDirty();
    193                t.transposeMetadata(axes);
    194                return t;
    195        }
    196
    197        @Override
    198        public Dataset transpose(int... axes) {
    199                Dataset t = getTransposedView(axes);
    200                return t == null ? clone() : t.clone();
    201        }
    202
    203        @Override
    204        public Dataset swapAxes(int axis1, int axis2) {
    205                int rank = shape.length;
    206                axis1 = ShapeUtils.checkAxis(rank, axis1);
    207                axis2 = ShapeUtils.checkAxis(rank, axis2);
    208
    209                if (rank == 1 || axis1 == axis2) {
    210                        return this;
    211                }
    212
    213                int[] axes = new int[rank];
    214                for (int i = 0; i < rank; i++) {
    215                        axes[i] = i;
    216                }
    217
    218                axes[axis1] = axis2;
    219                axes[axis2] = axis1;
    220                return getTransposedView(axes);
    221        }
    222
    223        boolean isContiguous() {
    224                if (stride == null)
    225                        return true;
    226
    227                if (offset != 0)
    228                        return false;
    229
    230                int s = getElementsPerItem();
    231                for (int j = getRank() - 1; j >= 0; j--) {
    232                        if (stride[j] != s) {
    233                                return false;
    234                        }
    235                        s *= shape[j];
    236                }
    237
    238                return true;
    239        }
    240
    241        @Override
    242        public Dataset flatten() {
    243                if (!isContiguous()) { // need to make a copy if not contiguous
    244                        return clone().flatten();
    245                }
    246                return reshape(size);
    247        }
    248
    249        /**
    250         * Fill dataset from object at depth dimension
    251         * @param obj
    252         * @param depth
    253         * @param pos position
    254         */
    255        protected void fillData(Object obj, final int depth, final int[] pos) {
    256                if (obj == null) {
    257                        Class<?> c = InterfaceUtils.getElementClass(getClass());
    258                        if (Float.class.equals(c)) {
    259                                set(Float.NaN, pos);
    260                        } else if (Float.class.equals(c)) {
    261                                set(Double.NaN, pos);
    262                        }
    263                        return;
    264                }
    265
    266                Class<?> clazz = obj.getClass();
    267                if (obj instanceof List<?>) {
    268                        List<?> jl = (List<?>) obj;
    269                        int l = jl.size();
    270                        for (int i = 0; i < l; i++) {
    271                                Object lo = jl.get(i);
    272                                fillData(lo, depth + 1, pos);
    273                                pos[depth]++;
    274                        }
    275                        pos[depth] = 0;
    276                } else if (clazz.isArray()) {
    277                        int l = Array.getLength(obj);
    278                        if (clazz.equals(odata.getClass())) {
    279                                System.arraycopy(obj, 0, odata, get1DIndex(pos), l);
    280                        } else if (clazz.getComponentType().isPrimitive()) {
    281                                for (int i = 0; i < l; i++) {
    282                                        set(Array.get(obj, i), pos);
    283                                        pos[depth]++;
    284                                }
    285                                pos[depth] = 0;
    286                        } else {
    287                                for (int i = 0; i < l; i++) {
    288                                        fillData(Array.get(obj, i), depth + 1, pos);
    289                                        pos[depth]++;
    290                                }
    291                                pos[depth] = 0;
    292                        }
    293                } else if (obj instanceof IDataset) {
    294                        boolean[] a = new boolean[shape.length];
    295                        Arrays.fill(a, depth, a.length, true);
    296                        setSlice(obj, getSliceIteratorFromAxes(pos, a));
    297                } else {
    298                        set(obj, pos);
    299                }
    300        }
    301
    302        @Override
    303        public IndexIterator getIterator(final boolean withPosition) {
    304                if (stride != null) {
    305                        return base.getSize() == 1  ? (withPosition ? new PositionIterator(offset, shape) :
    306                                new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset);
    307                }
    308                if (shape == null) {
    309                        return new NullIterator(shape, shape);
    310                }
    311
    312                return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size);
    313        }
    314
    315        @Override
    316        public IndexIterator getIterator() {
    317                return getIterator(false);
    318        }
    319
    320        @Override
    321        public PositionIterator getPositionIterator(final int... axes) {
    322                return new PositionIterator(shape, axes);
    323        }
    324
    325        @Override
    326        public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) {
    327                return getSliceIterator(new SliceND(shape, start, stop, step));
    328        }
    329
    330        /**
    331         * @param slice
    332         * @return an slice iterator that operates like an IndexIterator
    333         */
    334        public IndexIterator getSliceIterator(SliceND slice) {
    335                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
    336                        return new NullIterator(shape, slice.getShape());
    337                }
    338                if (stride != null) {
    339                        return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice);
    340                }
    341                return new SliceIterator(shape, size, slice);
    342        }
    343
    344        @Override
    345        public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) {
    346                int rank = shape.length;
    347                int[] start;
    348                int[] stop = new int[rank];
    349                int[] step = new int[rank];
    350
    351                if (pos == null) {
    352                        start = new int[rank];
    353                } else if (pos.length == rank) {
    354                        start = pos.clone();
    355                } else {
    356                        throw new IllegalArgumentException("pos array length is not equal to rank of dataset");
    357                }
    358                if (axes == null) {
    359                        axes = new boolean[rank];
    360                        Arrays.fill(axes, true);
    361                } else if (axes.length != rank) {
    362                        throw new IllegalArgumentException("axes array length is not equal to rank of dataset");
    363                }
    364
    365                for (int i = 0; i < rank; i++) {
    366                        if (axes[i]) {
    367                                stop[i] = shape[i];
    368                        } else {
    369                                stop[i] = start[i] + 1;
    370                        }
    371                        step[i] = 1;
    372                }
    373                return (SliceIterator) getSliceIterator(start, stop, step);
    374        }
    375
    376        @Override
    377        public BooleanIterator getBooleanIterator(Dataset choice) {
    378                return getBooleanIterator(choice, true);
    379        }
    380
    381        @Override
    382        public BooleanIterator getBooleanIterator(Dataset choice, boolean value) {
    383                return BooleanIterator.createIterator(value, this, choice, this);
    384        }
    385
    386        @Override
    387        public Dataset getByBoolean(Dataset selection) {
    388                checkCompatibility(selection);
    389
    390                final int length = ((Number) selection.sum()).intValue();
    391                final int is = getElementsPerItem();
    392                Dataset r = DatasetFactory.zeros(is, getClass(), length);
    393                BooleanIterator biter = getBooleanIterator(selection);
    394
    395                int i = 0;
    396                while (biter.hasNext()) {
    397                        r.setObjectAbs(i, getObjectAbs(biter.index));
    398                        i += is;
    399                }
    400                return r;
    401        }
    402
    403        @Override
    404        public Dataset getBy1DIndex(IntegerDataset index) {
    405                final int is = getElementsPerItem();
    406                final Dataset r = DatasetFactory.zeros(is, getClass(), index.getShapeRef());
    407                final IntegerIterator iter = new IntegerIterator(index, size, is);
    408
    409                int i = 0;
    410                while (iter.hasNext()) {
    411                        r.setObjectAbs(i, getObjectAbs(iter.index));
    412                        i += is;
    413                }
    414                return r;
    415        }
    416
    417        @Override
    418        public Dataset getByIndexes(final Object... indexes) {
    419                final IntegersIterator iter = new IntegersIterator(shape, indexes);
    420                final int is = getElementsPerItem();
    421                final Dataset r = DatasetFactory.zeros(is, getClass(), iter.getShape());
    422
    423                final int[] pos = iter.getPos();
    424                int i = 0;
    425                while (iter.hasNext()) {
    426                        r.setObjectAbs(i, getObject(pos));
    427                        i += is;
    428                }
    429                return r;
    430        }
    431
    432        @Override
    433        public boolean hasFloatingPointElements() {
    434                Class<?> cls = getElementClass();
    435                return cls == Float.class || cls == Double.class;
    436        }
    437
    438        @Override
    439        public int getElementsPerItem() {
    440                return DTypeUtils.getElementsPerItem(getDType());
    441        }
    442
    443        @Override
    444        public int getItemBytes() {
    445                return DTypeUtils.getItemBytes(getDType(), getElementsPerItem());
    446        }
    447
    448        @Override
    449        public int getSize() {
    450                return size;
    451        }
    452
    453        @Override
    454        public int[] getShape() {
    455                // make a copy of the dimensions data, and put that out
    456                if (shape == null) {
    457                        logger.warn("Shape is null!!!");
    458                        return new int[] {};
    459                }
    460                return shape.clone();
    461        }
    462
    463        @Override
    464        public int getRank() {
    465                return shape == null ? 0 : shape.length;
    466        }
    467
    468        @Override
    469        public int getNbytes() {
    470                return getSize() * getItemBytes();
    471        }
    472
    473        /**
    474         * Check for -1 placeholder in shape and replace if necessary
    475         * @param shape
    476         * @param size
    477         */
    478        private void checkShape(int[] shape, int size) {
    479                int rank = shape.length;
    480                int found = -1;
    481                int nsize = 1;
    482                for (int i = 0; i < rank; i++) {
    483                        int d = shape[i];
    484                        if (d == -1) {
    485                                if (found == -1) {
    486                                        found = i;
    487                                } else {
    488                                        logger.error("Can only have one -1 placeholder in shape");
    489                                        throw new IllegalArgumentException("Can only have one -1 placeholder in shape");
    490                                }
    491                        } else {
    492                                nsize *= d;
    493                        }
    494                }
    495                if (found >= 0) {
    496                        shape[found] = size/nsize;
    497                } else if (nsize != size && !(rank == 0 && size == 0)) {
    498                        logger.error("New shape is not same size as old shape");
    499                        throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape));
    500                }
    501        }
    502
    503        @Override
    504        public void setShape(final int... shape) {
    505                int[] nshape = shape.clone();
    506                checkShape(nshape, size);
    507                if (Arrays.equals(this.shape, nshape)) {
    508                        return;
    509                }
    510
    511                if (stride != null) {
    512                        // the only compatible shapes are ones where new dimensions are factors of old dimensions
    513                        // or are combined adjacent old dimensions
    514                        int[] oshape = this.shape;
    515                        int orank = oshape.length;
    516                        int nrank = nshape.length;
    517                        int diff = nrank - orank;
    518                        int[] nstride = new int[nrank];
    519                        boolean ones = true;
    520                        // work forwards for broadcasting cases
    521                        for (int i = 0, j = 0; i < orank || j < nrank;) {
    522                                if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) {
    523                                        nstride[j++] = stride[i++];
    524                                } else if (j < nrank && nshape[j] == 1) {
    525                                        nstride[j++] = 0;
    526                                } else if (i < orank && oshape[i] == 1) {
    527                                        i++;
    528                                } else {
    529                                        if (j < nrank)
    530                                                j++;
    531                                        if (i < orank)
    532                                                i++;
    533                                        ones = false;
    534                                }
    535                        }
    536                        if (!ones) { // not just ones differ in shapes
    537                                int[] ostride = stride;
    538                                int ob = 0;
    539                                int oe = 1;
    540                                int nb = 0;
    541                                int ne = 1;
    542                                while (ob < orank && nb < nrank) {
    543                                        int ol = oshape[ob];
    544                                        int nl = nshape[nb];
    545
    546                                        if (nl < ol) { // find group of shape dimensions that form common size
    547                                                do { // case where new shape spreads single dimension over several dimensions
    548                                                        if (ne == nrank) {
    549                                                                break;
    550                                                        }
    551                                                        nl *= nshape[ne++];
    552                                                } while (nl < ol);
    553                                                if (nl != ol) {
    554                                                        logger.error("Subshape is incompatible with single dimension");
    555                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
    556                                                }
    557                                                int on = ne - 1;
    558                                                while (nshape[on] == 1) {
    559                                                        on--;
    560                                                }
    561
    562                                                nstride[on] = ostride[ob];
    563                                                for (int n = on - 1; n >= nb; n--) {
    564                                                        if (nshape[n] == 1)
    565                                                                continue;
    566
    567                                                        nstride[n] = nshape[on] * nstride[on];
    568                                                        on = n;
    569                                                }
    570                                        } else if (ol < nl) {
    571                                                do { // case where new shape combines several dimensions into one dimension
    572                                                        if (oe == orank) {
    573                                                                break;
    574                                                        }
    575                                                        ol *= oshape[oe++];
    576                                                } while (ol < nl);
    577                                                if (nl != ol) {
    578                                                        logger.error("Single dimension is incompatible with subshape");
    579                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
    580                                                }
    581
    582                                                int oo = oe - 1;
    583                                                while (oshape[oo] == 1) {
    584                                                        oo--;
    585                                                }
    586                                                int os = ostride[oo];
    587                                                for (int o = oo - 1; o >= ob; o--) {
    588                                                        if (oshape[o] == 1)
    589                                                                continue;
    590                                                        if (ostride[o] != oshape[oo] * ostride[oo]) {
    591                                                                logger.error("Subshape cannot be a non-contiguous view");
    592                                                                throw new IllegalArgumentException("Subshape cannot be a non-contiguous view");
    593                                                        }
    594                                                        oo = o;
    595                                                }
    596                                                nstride[nb] = os;
    597                                        } else {
    598                                                nstride[nb] = ostride[ob];
    599                                        }
    600
    601                                        ob = oe++;
    602                                        nb = ne++;
    603                                }
    604                        }
    605
    606                        stride = nstride;
    607                }
    608
    609                setDirty();
    610                if (this.shape != null && metadata != null) {
    611                        reshapeMetadata(this.shape, nshape);
    612                }
    613                this.shape = nshape;
    614        }
    615
    616        @Override
    617        public int[] getShapeRef() {
    618                return shape;
    619        }
    620
    621        @Override
    622        public int getOffset() {
    623                return offset;
    624        }
    625
    626        @Override
    627        public int[] getStrides() {
    628                return stride;
    629        }
    630
    631        @Override
    632        public Serializable getBuffer() {
    633                return odata;
    634        }
    635
    636        @Override
    637        public void overrideInternal(Serializable buffer, int... shape) {
    638                if (buffer != null) {
    639                        odata = buffer;
    640                        setData();
    641                        setDirty();
    642                }
    643
    644                if (shape != null) {
    645                        this.shape = shape.clone();
    646                        size = ShapeUtils.calcSize(this.shape);
    647                }
    648        }
    649
    650        /**
    651         * Create a stride array from dataset
    652         * @param a dataset
    653         * @param offset output offset
    654         * @return new strides
    655         */
    656        public static int[] createStrides(Dataset a, final int[] offset) {
    657                return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset);
    658        }
    659
    660        /**
    661         * Create a stride array from dataset
    662         * @param isize
    663         * @param shape
    664         * @param oStride original stride
    665         * @param oOffset original offset (only used if there is an original stride)
    666         * @param offset output offset
    667         * @return new strides
    668         */
    669        public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) {
    670                int rank = shape.length;
    671                final int[] stride;
    672                if (oStride == null) {
    673                        offset[0] = 0;
    674                        stride = new int[rank];
    675                        int s = isize;
    676                        for (int j = rank - 1; j >= 0; j--) {
    677                                stride[j] = s;
    678                                s *= shape[j];
    679                        }
    680                } else {
    681                        offset[0] = oOffset;
    682                        stride = oStride.clone();
    683                }
    684                return stride;
    685        }
    686
    687        /**
    688         * Create a stride array from slice information and a dataset
    689         * @param slice
    690         * @param a dataset
    691         * @param stride output stride
    692         * @param offset output offset
    693         * @return new shape
    694         */
    695        public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) {
    696                return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset);
    697        }
    698
    699        /**
    700         * Create a stride array from slice and dataset information
    701         * @param slice
    702         * @param isize
    703         * @param shape
    704         * @param oStride original stride
    705         * @param oOffset original offset (only used if there is an original stride)
    706         * @param stride output stride
    707         * @param offset output offset
    708         * @return new shape
    709         */
    710        public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) {
    711                int[] lstart = slice.getStart();
    712                int[] lstep = slice.getStep();
    713                int[] newShape = slice.getShape();
    714                int rank = shape.length;
    715
    716                if (oStride == null) {
    717                        int s = isize;
    718                        offset[0] = 0;
    719                        for (int j = rank - 1; j >= 0; j--) {
    720                                stride[j] = s * lstep[j];
    721                                offset[0] += s * lstart[j];
    722                                s *= shape[j];
    723                        }
    724                } else {
    725                        offset[0] = oOffset;
    726                        for (int j = 0; j < rank; j++) {
    727                                int s = oStride[j];
    728                                stride[j] = lstep[j] * s;
    729                                offset[0] += lstart[j] * s;
    730                        }
    731                }
    732
    733                return newShape;
    734        }
    735
    736        @Override
    737        public Dataset getBroadcastView(int... broadcastShape) {
    738                AbstractDataset view = getView(true);
    739
    740                if (!Arrays.equals(shape, broadcastShape)) {
    741                        List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape);
    742                        view.setShape(nShapes.get(0));
    743                        view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape);
    744                        view.base = this;
    745                        view.shape = broadcastShape.clone();
    746                        view.size = ShapeUtils.calcSize(broadcastShape);
    747                        if (view.name == null || view.name.isEmpty()) {
    748                                view.name = "Broadcast from " + Arrays.toString(shape);
    749                        } else {
    750                                view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape);
    751                        }
    752                }
    753                return view;
    754        }
    755
    756        @Override
    757        public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) {
    758                return getSliceView(new SliceND(shape, start, stop, step));
    759        }
    760
    761        @Override
    762        public Dataset getSliceView(Slice... slice) {
    763                if (slice == null || slice.length == 0) {
    764                        return getView(true);
    765                }
    766
    767                return getSliceView(new SliceND(shape, slice));
    768        }
    769
    770        /**
    771         * Get a slice of the dataset. The returned dataset is a view on a selection of items
    772         * @param slice
    773         * @return slice view
    774         */
    775        @Override
    776        public Dataset getSliceView(SliceND slice) {
    777                if (slice.isAll()) {
    778                        return getView(true);
    779                }
    780
    781                final int rank = shape.length;
    782                int[] sStride = new int[rank];
    783                int[] sOffset = new int[1];
    784
    785                int[] sShape = createStrides(slice, this, sStride, sOffset);
    786
    787                AbstractDataset s = getView(false);
    788                s.shape = sShape;
    789                s.size = ShapeUtils.calcSize(sShape);
    790                s.stride = sStride;
    791                s.offset = sOffset[0];
    792                s.base = this;
    793
    794                s.metadata = copyMetadata();
    795                s.sliceMetadata(true, slice);
    796
    797                s.setDirty();
    798                s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE);
    799
    800                return s;
    801        }
    802
    803        /**
    804         * Get flattened view index of given position
    805         * @param pos
    806         *            the integer array specifying the n-D position
    807         * @return the index on the flattened dataset
    808         */
    809        private int getFlat1DIndex(final int[] pos) {
    810                final int imax = pos.length;
    811                if (imax == 0) {
    812                        return 0;
    813                }
    814
    815                return get1DIndexFromShape(pos);
    816        }
    817
    818        /**
    819         * @since 2.0
    820         */
    821        protected int getFirst1DIndex() {
    822                if (shape == null) {
    823                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    824                }
    825                return stride == null ? 0 : offset;
    826        }
    827
    828        @Override
    829        public int get1DIndex(final int... n) {
    830                if (n.length == 0 && shape.length == 0)
    831                        return offset;
    832
    833                return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n);
    834        }
    835
    836        private static void throwAIOOBException(int i, int s, int d) {
    837                throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s
    838                                + "] in dimension " + d);
    839        }
    840
    841        /**
    842         * @param i
    843         * @return the index on the data array corresponding to that location
    844         */
    845        protected int get1DIndex(int i) {
    846                if (shape == null) {
    847                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    848                }
    849                if (shape.length > 1) {
    850                        logger.error("This dataset is not 1D but was addressed as such");
    851                        throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such");
    852                }
    853                if (i < 0) {
    854                        i += shape[0];
    855                }
    856                if (i < 0 || i >= shape[0]) {
    857                        throwAIOOBException(i, shape[0], 0);
    858                }
    859                return stride == null ? i : i*stride[0] + offset;
    860        }
    861
    862        /**
    863         * @param i
    864         * @param j
    865         * @return the index on the data array corresponding to that location
    866         */
    867        protected int get1DIndex(int i, int j) {
    868                if (shape == null) {
    869                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    870                }
    871                if (shape.length != 2) {
    872                        logger.error("This dataset is not 2D but was addressed as such");
    873                        throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such");
    874                }
    875                if (i < 0) {
    876                        i += shape[0];
    877                }
    878                if (i < 0 || i >= shape[0]) {
    879                        throwAIOOBException(i, shape[0], 0);
    880                }
    881                if (j < 0) {
    882                        j += shape[1];
    883                }
    884                if (j < 0 || j >= shape[1]) {
    885                        throwAIOOBException(i, shape[1], 1);
    886                }
    887                return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset;
    888        }
    889
    890        protected int get1DIndexFromShape(final int[] n) {
    891                return get1DIndexFromShape(shape, n);
    892        }
    893
    894        protected static int get1DIndexFromShape(final int[] shape, final int[] n) {
    895                if (shape == null) {
    896                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    897                }
    898                final int rank = shape.length;
    899                if (rank != n.length) {
    900                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
    901                        logger.error(errMsg);
    902                        throw new IllegalArgumentException(errMsg);
    903                }
    904                int index = 0;
    905                for (int i = 0; i < rank; i++) {
    906                        final int si = shape[i];
    907                        int ni = n[i];
    908                        if (ni < 0) {
    909                                ni += si;
    910                        }
    911                        if (ni < 0 || ni >= si) {
    912                                throwAIOOBException(ni, si, i);
    913                        }
    914                        index = index * si + ni;
    915                }
    916
    917                return index;
    918        }
    919
    920        private int get1DIndexFromStrides(final int[] n) {
    921                return get1DIndexFromStrides(shape, stride, offset, n);
    922        }
    923
    924        private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) {
    925                if (shape == null) {
    926                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    927                }
    928                final int rank = shape.length;
    929                if (rank != n.length) {
    930                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
    931                        logger.error(errMsg);
    932                        throw new IllegalArgumentException(errMsg);
    933                }
    934                int index = offset;
    935                for (int i = 0; i < rank; i++) {
    936                        final int st = stride[i];
    937                        if (st != 0) { // not broadcasted
    938                                final int si = shape[i];
    939                                int ni = n[i];
    940                                if (ni < 0) {
    941                                        ni += si;
    942                                }
    943                                if (ni < 0 || ni >= si) {
    944                                        throwAIOOBException(ni, si, i);
    945                                }
    946                                index += st * ni;
    947                        }
    948                }
    949                return index;
    950        }
    951
    952        @Override
    953        public int[] getNDPosition(final int n) {
    954                if (isIndexInRange(n)) {
    955                        throw new IllegalArgumentException("Index provided " + n
    956                                        + "is larger then the size of the containing array");
    957                }
    958
    959                return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n);
    960        }
    961
    962        private boolean isIndexInRange(final int n) {
    963                if (stride == null) {
    964                        return n >= size;
    965                }
    966                return n >= getBufferLength();
    967        }
    968
    969        /**
    970         * @return entire buffer length
    971         */
    972        abstract protected int getBufferLength();
    973
    974        private int[] getNDPositionFromStrides(int n) {
    975                n -= offset;
    976                int rank = shape.length;
    977                if (rank == 1) {
    978                        return new int[] { n / stride[0] };
    979                }
    980
    981                int[] output = new int[rank];
    982                int i = 0;
    983                while (i != n) { // TODO find more efficient way than this exhaustive search
    984                        int j = rank - 1;
    985                        for (; j >= 0; j--) {
    986                                output[j]++;
    987                                i += stride[j];
    988                                if (output[j] >= shape[j]) {
    989                                        output[j] = 0;
    990                                        i -= shape[j] * stride[j];
    991                                } else {
    992                                        break;
    993                                }
    994                        }
    995                        if (j == -1) {
    996                                logger.error("Index was not found in this strided dataset");
    997                                throw new IllegalArgumentException("Index was not found in this strided dataset");
    998                        }
    999                }
    1000
    1001                return output;
    1002        }
    1003
    1004        @Override
    1005        public int checkAxis(int axis) {
    1006                return ShapeUtils.checkAxis(shape.length, axis);
    1007        }
    1008
    1009        @Deprecated
    1010        protected static int checkAxis(int rank, int axis) {
    1011                return ShapeUtils.checkAxis(rank, axis);
    1012        }
    1013
    1014        protected static final char BLOCK_OPEN = '[';
    1015        protected static final char BLOCK_CLOSE = ']';
    1016
    1017        @Override
    1018        public String toString() {
    1019                final int rank = shape == null ? 0 : shape.length;
    1020                final StringBuilder out = new StringBuilder();
    1021
    1022                if (DTypeUtils.isDTypeElemental(getDType())) {
    1023                        out.append("Dataset ");
    1024                } else {
    1025                        out.append("Compound dataset (");
    1026                        out.append(getElementsPerItem());
    1027                        out.append(") ");
    1028                }
    1029
    1030                if (name != null && name.length() > 0) {
    1031                        out.append("'");
    1032                        out.append(name);
    1033                        out.append("' has shape ");
    1034                } else {
    1035                        out.append("shape is ");
    1036                }
    1037
    1038                out.append(BLOCK_OPEN);
    1039                if (rank > 0) {
    1040                        out.append(shape[0]);
    1041                }
    1042                for (int i = 1; i < rank; i++) {
    1043                        out.append(", " + shape[i]);
    1044                }
    1045                out.append(BLOCK_CLOSE);
    1046                return out.toString();
    1047        }
    1048
    1049        @Override
    1050        public String toString(boolean showData) {
    1051                if (!showData) {
    1052                        return toString();
    1053                }
    1054
    1055                if (size == 0) {
    1056                        return "[]";
    1057                }
    1058
    1059                final int rank = shape == null ? 0 : shape.length;
    1060                final StringBuilder out = new StringBuilder();
    1061
    1062                if (rank > 0) {
    1063                        int[] pos = new int[rank];
    1064                        final StringBuilder lead = new StringBuilder();
    1065                        printBlocks(out, lead, 0, pos);
    1066                } else {
    1067                        out.append(getString());
    1068                }
    1069                return out.toString();
    1070        }
    1071
    1072        /**
    1073         * Limit to strings output via the toString() method
    1074         */
    1075        private static int maxStringLength = 120;
    1076
    1077        /**
    1078         * Set maximum line length for toString() method
    1079         * @param maxLineLength
    1080         */
    1081        public static void setMaxLineLength(int maxLineLength) {
    1082                maxStringLength = maxLineLength;
    1083        }
    1084
    1085        /**
    1086         * @return maximum line length for toString() method
    1087         */
    1088        public static int getMaxLineLength() {
    1089                return maxStringLength;
    1090        }
    1091
    1092        /**
    1093         * Limit to number of sub-blocks output via the toString() method
    1094         */
    1095        private static final int MAX_SUBBLOCKS = 6;
    1096
    1097        private final static String SEPARATOR = ",";
    1098        private final static String SPACE = " ";
    1099        private final static String ELLIPSIS = "...";
    1100        private final static String NEWLINE = "\n";
    1101
    1102        /**
    1103         * Make a line of output for last dimension of dataset
    1104         *
    1105         * @param start
    1106         * @return line
    1107         */
    1108        private StringBuilder makeLine(final int end, final int[] start) {
    1109                StringBuilder line = new StringBuilder();
    1110                final int[] pos;
    1111                if (end >= start.length) {
    1112                        pos = Arrays.copyOf(start, end + 1);
    1113                } else {
    1114                        pos = start;
    1115                }
    1116                pos[end] = 0;
    1117                line.append(BLOCK_OPEN);
    1118                line.append(getString(pos));
    1119
    1120                final int length = shape[end];
    1121
    1122                // trim elements printed if length exceed estimate of maximum elements
    1123                int excess = length - maxStringLength / 3; // space + number + separator
    1124                int midIndex = -1;
    1125                if (excess > 0) {
    1126                        int index = (length - excess) / 2;
    1127                        for (int y = 1; y < index; y++) {
    1128                                line.append(SEPARATOR + SPACE);
    1129                                pos[end] = y;
    1130                                line.append(getString(pos));
    1131                        }
    1132                        midIndex = line.length() + 2;
    1133                        index = (length + excess) / 2;
    1134                        for (int y = index; y < length; y++) {
    1135                                line.append(SEPARATOR + SPACE);
    1136                                pos[end] = y;
    1137                                line.append(getString(pos));
    1138                        }
    1139                } else {
    1140                        for (int y = 1; y < length; y++) {
    1141                                line.append(SEPARATOR + SPACE);
    1142                                pos[end] = y;
    1143                                line.append(getString(pos));
    1144                        }
    1145                }
    1146                line.append(BLOCK_CLOSE);
    1147
    1148                // trim string down to limit
    1149                int lineLength = line.length();
    1150                excess = lineLength - maxStringLength - ELLIPSIS.length() - 1;
    1151                if (excess > 0) {
    1152                        int index = (lineLength - excess) / 2;
    1153                        if (midIndex > 0 && index > midIndex) {
    1154                                index = midIndex;
    1155                        } else {
    1156                                index = line.lastIndexOf(SEPARATOR, index) + 2;
    1157                        }
    1158                        StringBuilder out = new StringBuilder(line.subSequence(0, index));
    1159                        out.append(ELLIPSIS + SEPARATOR);
    1160                        index = (lineLength + excess) / 2;
    1161                        if (midIndex > 0 && index <= midIndex) {
    1162                                index = midIndex - 1;
    1163                        } else {
    1164                                index = line.indexOf(SEPARATOR, index) + 1;
    1165                        }
    1166                        out.append(line.subSequence(index, lineLength));
    1167                        return out;
    1168                } else if (midIndex > 0) { // add ellipsis
    1169                        StringBuilder out = new StringBuilder(line.subSequence(0, midIndex));
    1170                        out.append(ELLIPSIS + SEPARATOR + SPACE);
    1171                        out.append(line.subSequence(midIndex, lineLength));
    1172                        return out;
    1173                }
    1174
    1175                return line;
    1176        }
    1177
    1178        /**
    1179         * recursive method to print blocks
    1180         */
    1181        private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) {
    1182                if (out.length() > 0) {
    1183                        char last = out.charAt(out.length() - 1);
    1184                        if (last != BLOCK_OPEN) {
    1185                                out.append(lead);
    1186                        }
    1187                }
    1188                final int end = getRank() - 1;
    1189                if (level != end) {
    1190                        out.append(BLOCK_OPEN);
    1191                        int length = shape[level];
    1192
    1193                        // first sub-block
    1194                        pos[level] = 0;
    1195                        StringBuilder newlead = new StringBuilder(lead);
    1196                        newlead.append(SPACE);
    1197                        printBlocks(out, newlead, level + 1, pos);
    1198                        if (length < 2) { // escape
    1199                                out.append(BLOCK_CLOSE);
    1200                                return;
    1201                        }
    1202
    1203                        out.append(SEPARATOR + NEWLINE);
    1204                        for (int i = level + 1; i < end; i++) {
    1205                                out.append(NEWLINE);
    1206                        }
    1207
    1208                        // middle sub-blocks
    1209                        if (length < MAX_SUBBLOCKS) {
    1210                                for (int x = 1; x < length - 1; x++) {
    1211                                        pos[level] = x;
    1212                                        printBlocks(out, newlead, level + 1, pos);
    1213                                        if (end <= level + 1) {
    1214                                                out.append(SEPARATOR + NEWLINE);
    1215                                        } else {
    1216                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
    1217                                        }
    1218                                }
    1219                        } else {
    1220                                final int excess = length - MAX_SUBBLOCKS;
    1221                                int xmax = (length - excess) / 2;
    1222                                for (int x = 1; x < xmax; x++) {
    1223                                        pos[level] = x;
    1224                                        printBlocks(out, newlead, level + 1, pos);
    1225                                        if (end <= level + 1) {
    1226                                                out.append(SEPARATOR + NEWLINE);
    1227                                        } else {
    1228                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
    1229                                        }
    1230                                }
    1231                                out.append(newlead);
    1232                                out.append(ELLIPSIS + SEPARATOR + NEWLINE);
    1233                                xmax = (length + excess) / 2;
    1234                                for (int x = xmax; x < length - 1; x++) {
    1235                                        pos[level] = x;
    1236                                        printBlocks(out, newlead, level + 1, pos);
    1237                                        if (end <= level + 1) {
    1238                                                out.append(SEPARATOR + NEWLINE);
    1239                                        } else {
    1240                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
    1241                                        }
    1242                                }
    1243                        }
    1244
    1245                        // last sub-block
    1246                        pos[level] = length - 1;
    1247                        printBlocks(out, newlead, level + 1, pos);
    1248                        out.append(BLOCK_CLOSE);
    1249                } else {
    1250                        out.append(makeLine(end, pos));
    1251                }
    1252        }
    1253
    1254        @Override
    1255        public Dataset squeezeEnds() {
    1256                return squeeze(true);
    1257        }
    1258
    1259        @Override
    1260        public Dataset squeeze() {
    1261                return squeeze(false);
    1262        }
    1263
    1264        @Override
    1265        public Dataset squeeze(boolean onlyFromEnds) {
    1266                final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds);
    1267                final int[] oshape = shape;
    1268                if (stride == null) {
    1269                        shape = tshape;
    1270                } else {
    1271                        int rank = shape.length;
    1272                        int trank = tshape.length;
    1273                        if (trank < rank) {
    1274                                int[] tstride = new int[tshape.length];
    1275                                if (onlyFromEnds) {
    1276                                        for (int i = 0; i < rank; i++) {
    1277                                                if (shape[i] != 1) {
    1278                                                        for (int k = 0; k < trank; k++) {
    1279                                                                tstride[k] = stride[i++];
    1280                                                        }
    1281                                                        break;
    1282                                                }
    1283                                        }
    1284                                } else {
    1285                                        int t = 0;
    1286                                        for (int i = 0; i < rank; i++) {
    1287                                                if (shape[i] != 1) {
    1288                                                        tstride[t++] = stride[i];
    1289                                                }
    1290                                        }
    1291                                }
    1292                                shape = tshape;
    1293                                stride = tstride;
    1294                        }
    1295                }
    1296
    1297                setDirty();
    1298                reshapeMetadata(oshape, shape);
    1299                return this;
    1300        }
    1301
    1302        @Override
    1303        public boolean isCompatibleWith(final ILazyDataset g) {
    1304                return ShapeUtils.areShapesCompatible(shape, g.getShape());
    1305        }
    1306
    1307        @Override
    1308        public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException {
    1309                ShapeUtils.checkCompatibility(this, g);
    1310        }
    1311
    1312        @Override
    1313        public Dataset reshape(final int... shape) {
    1314                Dataset a;
    1315                try {
    1316                        a = getView(true);
    1317                        a.setShape(shape);
    1318                } catch (IllegalArgumentException e) {
    1319                        a = clone();
    1320                        a.setShape(shape);
    1321                }
    1322                return a;
    1323        }
    1324
    1325        /**
    1326         * @param start
    1327         * @param stop
    1328         * @param step
    1329         * @return number of steps to take
    1330         */
    1331        protected static int calcSteps(final double start, final double stop, final double step) {
    1332                return Math.max(0, (int) Math.ceil((stop - start) / step));
    1333        }
    1334
    1335        @Override
    1336        public boolean isComplex() {
    1337                int type = getDType();
    1338                return type == COMPLEX64 || type == COMPLEX128;
    1339        }
    1340
    1341        @Override
    1342        public Dataset getRealPart() {
    1343                return this;
    1344        }
    1345
    1346        @Override
    1347        public Dataset getRealView() {
    1348                return getView(true);
    1349        }
    1350
    1351        @Override
    1352        public Dataset getSlice(final int[] start, final int[] stop, final int[] step) {
    1353                return getSlice(new SliceND(shape, start, stop, step));
    1354        }
    1355
    1356        @Override
    1357        public Dataset getSlice(Slice... slice) {
    1358                return getSlice(new SliceND(shape, slice));
    1359        }
    1360
    1361        @Override
    1362        public Dataset getSlice(IMonitor monitor, Slice... slice) {
    1363                return getSlice(slice);
    1364        }
    1365
    1366        @Override
    1367        public Dataset getSlice(IMonitor monitor, SliceND slice) {
    1368                return getSlice(slice);
    1369        }
    1370
    1371        @Override
    1372        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) {
    1373                return getSlice(start, stop, step);
    1374        }
    1375
    1376        /**
    1377         * Get a slice of the dataset. The returned dataset is a copied selection of items
    1378         * @param slice
    1379         * @return The dataset of the sliced data
    1380         */
    1381        @Override
    1382        public Dataset getSlice(final SliceND slice) {
    1383                SliceIterator it = (SliceIterator) getSliceIterator(slice);
    1384                AbstractDataset s = getSlice(it);
    1385                s.metadata = copyMetadata();
    1386                s.setDirty();
    1387                s.sliceMetadata(true, slice);
    1388                return s;
    1389        }
    1390
    1391        /**
    1392         * Get a slice of the dataset. The returned dataset is a copied selection of items
    1393         *
    1394         * @param iterator Slice iterator
    1395         * @return The dataset of the sliced data
    1396         */
    1397        abstract public AbstractDataset getSlice(final SliceIterator iterator);
    1398
    1399        @Override
    1400        public Dataset setSlice(final Object obj, final SliceND slice) {
    1401                Dataset ds;
    1402                if (obj instanceof Dataset) {
    1403                        ds = (Dataset) obj;
    1404                } else if (obj instanceof IDataset) {
    1405                        ds = DatasetUtils.convertToDataset((IDataset) obj);
    1406                } else {
    1407                        Class<? extends Dataset> dClass = getClass();
    1408                        if (!BooleanDataset.class.equals(dClass)) {
    1409                                dClass = DTypeUtils.getLargestDataset(dClass);
    1410                        }
    1411                        ds = DatasetFactory.createFromObject(getElementsPerItem(), dClass, obj);
    1412                }
    1413
    1414                return setSlicedView(getSliceView(slice), ds);
    1415        }
    1416
    1417        @Override
    1418        public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) {
    1419                return setSlice(obj, new SliceND(shape, start, stop, step));
    1420        }
    1421
    1422        /**
    1423         * Set a view of current dataset to given dataset with broadcasting
    1424         * @param view
    1425         * @param d
    1426         * @return this dataset
    1427         */
    1428        abstract Dataset setSlicedView(Dataset view, Dataset d);
    1429
    1430        @Override
    1431        public Dataset setSlice(Object obj, Slice... slice) {
    1432                if (slice == null || slice.length == 0) {
    1433                        return setSlice(obj, new SliceND(shape));
    1434                }
    1435                return setSlice(obj, new SliceND(shape, slice));
    1436        }
    1437
    1438        @Override
    1439        public boolean all() {
    1440                return Comparisons.allTrue(this);
    1441        }
    1442
    1443        @Override
    1444        public BooleanDataset all(final int axis) {
    1445                return Comparisons.allTrue(this, axis);
    1446        }
    1447
    1448        @Override
    1449        public boolean any() {
    1450                return Comparisons.anyTrue(this);
    1451        }
    1452
    1453        @Override
    1454        public BooleanDataset any(final int axis) {
    1455                return Comparisons.anyTrue(this, axis);
    1456        }
    1457
    1458        @Override
    1459        public Dataset ifloorDivide(final Object o) {
    1460                return idivide(o).ifloor();
    1461        }
    1462
    1463        @Override
    1464        public double residual(final Object o) {
    1465                return residual(o, null, false);
    1466        }
    1467
    1468        @Override
    1469        public double residual(final Object o, boolean ignoreNaNs) {
    1470                return residual(o, null, ignoreNaNs);
    1471        }
    1472
    1473        /**
    1474         * @since 2.0
    1475         */
    1476        @SuppressWarnings("unchecked")
    1477        protected StatisticsMetadata<Number> getStats() {
    1478                StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class);
    1479                if (md == null || md.isDirty(this)) {
    1480                        md = new StatisticsMetadataImpl<Number>();
    1481                        md.initialize(this);
    1482                        setMetadata(md);
    1483                }
    1484                return md;
    1485        }
    1486
    1487        /**
    1488         * @since 2.0
    1489         */
    1490        @SuppressWarnings("unchecked")
    1491        protected StatisticsMetadata<String> getStringStats() {
    1492                StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class);
    1493                if (md == null || md.isDirty(this)) {
    1494                        md = new StatisticsMetadataImpl<String>();
    1495                        md.initialize(this);
    1496                        setMetadata(md);
    1497                }
    1498                return md;
    1499        }
    1500
    1501        @Override
    1502        public Number max(boolean... ignoreInvalids) {
    1503                return getStats().getMaximum(ignoreInvalids);
    1504        }
    1505
    1506        @Override
    1507        public Dataset max(int axis, boolean... ignoreInvalids) {
    1508                return getStats().getMaximum(axis, ignoreInvalids);
    1509        }
    1510
    1511        @Override
    1512        public Dataset max(int[] axes, boolean... ignoreInvalids) {
    1513                return getStats().getMaximum(axes, ignoreInvalids);
    1514        }
    1515
    1516        @Override
    1517        public Number min(boolean... ignoreInvalids) {
    1518                return getStats().getMinimum(ignoreInvalids);
    1519        }
    1520
    1521        @Override
    1522        public Dataset min(int axis, boolean... ignoreInvalids) {
    1523                return getStats().getMinimum(axis, ignoreInvalids);
    1524        }
    1525
    1526        @Override
    1527        public Dataset min(int[] axes, boolean... ignoreInvalids) {
    1528                return getStats().getMinimum(axes, ignoreInvalids);
    1529        }
    1530
    1531        @Override
    1532        public int argMax(boolean... ignoreInvalids) {
    1533                return getFlat1DIndex(maxPos(ignoreInvalids));
    1534        }
    1535
    1536        /**
    1537         * @since 2.0
    1538         */
    1539        @Override
    1540        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
    1541                return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids);
    1542        }
    1543
    1544        @Override
    1545        public int argMin(boolean... ignoreInvalids) {
    1546                return getFlat1DIndex(minPos(ignoreInvalids));
    1547        }
    1548
    1549        /**
    1550         * @since 2.0
    1551         */
    1552        @Override
    1553        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
    1554                return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids);
    1555        }
    1556
    1557        @Override
    1558        public Number peakToPeak(boolean... ignoreInvalids) {
    1559                return InterfaceUtils.fromDoubleToBiggestNumber(getClass(), max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue());
    1560        }
    1561
    1562        @Override
    1563        public Dataset peakToPeak(int axis,  boolean... ignoreInvalids) {
    1564                return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids));
    1565        }
    1566
    1567        @Override
    1568        public Dataset peakToPeak(int[] axes,  boolean... ignoreInvalids) {
    1569                return Maths.subtract(max(axes, ignoreInvalids), min(axes, ignoreInvalids));
    1570        }
    1571
    1572        @Override
    1573        public long count(boolean... ignoreInvalids) {
    1574                return getStats().getCount(ignoreInvalids);
    1575        }
    1576
    1577        @Override
    1578        public Dataset count(int axis, boolean... ignoreInvalids) {
    1579                return getStats().getCount(axis, ignoreInvalids);
    1580        }
    1581
    1582        @Override
    1583        public Dataset count(int[] axes, boolean... ignoreInvalids) {
    1584                return getStats().getCount(axes, ignoreInvalids);
    1585        }
    1586
    1587        @Override
    1588        public Object sum(boolean... ignoreInvalids) {
    1589                return InterfaceUtils.toBiggestNumber(getClass(), getStats().getSum(ignoreInvalids));
    1590        }
    1591
    1592        @Override
    1593        public Dataset sum(int axis, boolean... ignoreInvalids) {
    1594                return getStats().getSum(axis, ignoreInvalids);
    1595        }
    1596
    1597        @Override
    1598        public Dataset sum(int[] axes, boolean... ignoreInvalids) {
    1599                return getStats().getSum(axes, ignoreInvalids);
    1600        }
    1601
    1602        @Override
    1603        public Object product(boolean... ignoreInvalids) {
    1604                return Stats.product(this, ignoreInvalids);
    1605        }
    1606
    1607        @Override
    1608        public Dataset product(int axis, boolean... ignoreInvalids) {
    1609                return Stats.product(this, axis, ignoreInvalids);
    1610        }
    1611
    1612        @Override
    1613        public Dataset product(int[] axes, boolean... ignoreInvalids) {
    1614                return Stats.product(this, axes, ignoreInvalids);
    1615        }
    1616
    1617        @Override
    1618        public Object mean(boolean... ignoreInvalids) {
    1619                return getStats().getMean(ignoreInvalids);
    1620        }
    1621
    1622        @Override
    1623        public Dataset mean(int axis, boolean... ignoreInvalids) {
    1624                return getStats().getMean(axis, ignoreInvalids);
    1625        }
    1626
    1627        @Override
    1628        public Dataset mean(int[] axes, boolean... ignoreInvalids) {
    1629                return getStats().getMean(axes, ignoreInvalids);
    1630        }
    1631
    1632        @Override
    1633        public double variance() {
    1634                return variance(false);
    1635        }
    1636
    1637        @Override
    1638        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
    1639                return getStats().getVariance(isWholePopulation, ignoreInvalids);
    1640        }
    1641
    1642        @Override
    1643        public Dataset variance(int axis) {
    1644                return getStats().getVariance(axis, false);
    1645        }
    1646
    1647        @Override
    1648        public Dataset variance(int[] axes) {
    1649                return getStats().getVariance(axes, false);
    1650        }
    1651
    1652        @Override
    1653        public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    1654                return getStats().getVariance(axis, isWholePopulation, ignoreInvalids);
    1655        }
    1656
    1657        @Override
    1658        public Dataset variance(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) {
    1659                return getStats().getVariance(axes, isWholePopulation, ignoreInvalids);
    1660        }
    1661
    1662        @Override
    1663        public double stdDeviation() {
    1664                return Math.sqrt(variance());
    1665        }
    1666
    1667        @Override
    1668        public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) {
    1669                return Math.sqrt(variance(isWholePopulation, ignoreInvalids));
    1670        }
    1671
    1672        @Override
    1673        public Dataset stdDeviation(int axis) {
    1674                return Maths.sqrt(variance(axis, false));
    1675        }
    1676
    1677        @Override
    1678        public Dataset stdDeviation(int[] axes) {
    1679                return Maths.sqrt(variance(axes, false));
    1680        }
    1681
    1682        @Override
    1683        public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    1684                return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids));
    1685        }
    1686
    1687        @Override
    1688        public Dataset stdDeviation(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) {
    1689                return Maths.sqrt(variance(axes, isWholePopulation, ignoreInvalids));
    1690        }
    1691
    1692        @Override
    1693        public double rootMeanSquare(boolean... ignoreInvalids) {
    1694                StatisticsMetadata<Number> stats = getStats();
    1695                final double mean = stats.getMean(ignoreInvalids).doubleValue();
    1696                final double var = stats.getVariance(true, ignoreInvalids);
    1697                return Math.sqrt(var + mean * mean);
    1698        }
    1699
    1700        @Override
    1701        public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
    1702                StatisticsMetadata<Number> stats = getStats();
    1703                Dataset v = stats.getVariance(axis, true, ignoreInvalids);
    1704                Dataset m = stats.getMean(axis, ignoreInvalids);
    1705                Dataset result = Maths.multiply(m, m);
    1706                return Maths.sqrt(result.iadd(v));
    1707        }
    1708
    1709        @Override
    1710        public Dataset rootMeanSquare(int[] axes, boolean... ignoreInvalids) {
    1711                StatisticsMetadata<Number> stats = getStats();
    1712                Dataset v = stats.getVariance(axes, true, ignoreInvalids);
    1713                Dataset m = stats.getMean(axes, ignoreInvalids);
    1714                Dataset result = Maths.multiply(m, m);
    1715                return Maths.sqrt(result.iadd(v));
    1716        }
    1717
    1718        /**
    1719         * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards.
    1720         *
    1721         * @param dindex
    1722         * @param sindex
    1723         * @param src
    1724         *            is the source data buffer
    1725         */
    1726        protected abstract void setItemDirect(final int dindex, final int sindex, final Object src);
    1727
    1728        /**
    1729         * @return error broadcasted to current shape
    1730         */
    1731        private Dataset getBroadcastedInternalError() {
    1732                if (shape == null) {
    1733                        throw new IllegalArgumentException("Cannot get error for null dataset");
    1734                }
    1735                ILazyDataset led = super.getErrors();
    1736                if (led == null)
    1737                        return null;
    1738
    1739                Dataset ed = null;
    1740                try {
    1741                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
    1742                } catch (DatasetException e) {
    1743                        logger.error("Could not get data from lazy dataset", e);
    1744                }
    1745                if (led != ed) {
    1746                        setErrors(ed); // set back
    1747                }
    1748
    1749                return ed.getBroadcastView(shape);
    1750        }
    1751
    1752        @Override
    1753        public Dataset getErrors() {
    1754                Dataset ed = getBroadcastedInternalError();
    1755                if (ed == null)
    1756                        return null;
    1757
    1758                return ed;
    1759        }
    1760
    1761        @Override
    1762        public double getError() {
    1763                Dataset ed = getBroadcastedInternalError();
    1764                if (ed == null)
    1765                        return 0;
    1766
    1767                return ed.getDouble();
    1768        }
    1769
    1770        @Override
    1771        public double getError(final int i) {
    1772                Dataset ed = getBroadcastedInternalError();
    1773                if (ed == null)
    1774                        return 0;
    1775
    1776                return ed.getDouble(i);
    1777        }
    1778
    1779        @Override
    1780        public double getError(final int i, final int j) {
    1781                Dataset ed = getBroadcastedInternalError();
    1782                if (ed == null)
    1783                        return 0;
    1784
    1785                return ed.getDouble(i, j);
    1786        }
    1787
    1788        @Override
    1789        public double getError(int... pos) {
    1790                Dataset ed = getBroadcastedInternalError();
    1791                if (ed == null)
    1792                        return 0;
    1793
    1794                return ed.getDouble(pos);
    1795        }
    1796
    1797        @Override
    1798        public double[] getErrorArray(final int i) {
    1799                Dataset ed = getBroadcastedInternalError();
    1800                if (ed == null)
    1801                        return null;
    1802
    1803                return new double[] {getError(i)};
    1804        }
    1805
    1806        @Override
    1807        public double[] getErrorArray(final int i, final int j) {
    1808                Dataset ed = getBroadcastedInternalError();
    1809                if (ed == null)
    1810                        return null;
    1811
    1812                return new double[] {getError(i, j)};
    1813        }
    1814
    1815        @Override
    1816        public double[] getErrorArray(int... pos) {
    1817                Dataset ed = getBroadcastedInternalError();
    1818                if (ed == null)
    1819                        return null;
    1820
    1821                return new double[] {getError(pos)};
    1822        }
    1823
    1824        protected Dataset getInternalSquaredError() {
    1825                Dataset sed = getErrorBuffer().getBroadcastView(shape);
    1826                return sed;
    1827        }
    1828
    1829        @Override
    1830        public Dataset getErrorBuffer() {
    1831                ErrorMetadata emd = getErrorMetadata();
    1832                if (emd == null)
    1833                        return null;
    1834
    1835                if (!(emd instanceof ErrorMetadataImpl)) {
    1836                        ILazyDataset led = emd.getError();
    1837                        Dataset ed;
    1838                        try {
    1839                                ed = DatasetUtils.sliceAndConvertLazyDataset(led);
    1840                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1841                                setMetadata(emd);
    1842                                emd.setError(ed);
    1843                        } catch (MetadataException me) {
    1844                                logger.error("Could not create metadata", me);
    1845                        } catch (DatasetException e) {
    1846                                logger.error("Could not get data from lazy dataset", e);
    1847                        }
    1848                }
    1849
    1850                return ((ErrorMetadataImpl) emd).getSquaredError();
    1851        }
    1852
    1853        /**
    1854         * Set a copy of the buffer that backs the (squared) error data
    1855         * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset
    1856         */
    1857        @Override
    1858        public void setErrorBuffer(Serializable buffer) {
    1859                if (shape == null) {
    1860                        throw new IllegalArgumentException("Cannot set error buffer for null dataset");
    1861                }
    1862                if (buffer == null) {
    1863                        clearMetadata(ErrorMetadata.class);
    1864                        return;
    1865                }
    1866
    1867                IDataset d = (IDataset) createFromSerializable(buffer, false);
    1868                ErrorMetadata emd = getErrorMetadata();
    1869                if (!(emd instanceof ErrorMetadataImpl)) {
    1870                        try {
    1871                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1872                                setMetadata(emd);
    1873                        } catch (MetadataException me) {
    1874                                logger.error("Could not create metadata", me);
    1875                        }
    1876                }
    1877                ((ErrorMetadataImpl) emd).setSquaredError(d);
    1878        }
    1879}