    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.Serializable;
    016import java.lang.reflect.Array;
    017import java.text.Format;
    018import java.util.ArrayList;
    019import java.util.Arrays;
    020import java.util.List;
    021import java.util.concurrent.ConcurrentHashMap;
    022import java.util.concurrent.ConcurrentMap;
    023
    024import org.eclipse.january.DatasetException;
    025import org.eclipse.january.IMonitor;
    026import org.eclipse.january.MetadataException;
    027import org.eclipse.january.metadata.ErrorMetadata;
    028import org.eclipse.january.metadata.MetadataFactory;
    029import org.eclipse.january.metadata.MetadataType;
    030import org.eclipse.january.metadata.StatisticsMetadata;
    031import org.eclipse.january.metadata.internal.ErrorMetadataImpl;
    032import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
    033
    034/**
    035 * Generic container class for data
    036 * <p/>
    037 * Each subclass has an array of primitive types, elements of this array are grouped or
    038 * compounded to make items
    039 * <p/>
    040 * Data items can be boolean, integer, float, complex float, vector float, etc
    041 */
    042public abstract class AbstractDataset extends LazyDatasetBase implements Dataset {
    043        // pin UID to base class
    044        private static final long serialVersionUID = Dataset.serialVersionUID;
    045
    046        protected int size; // number of items
    047
    048        transient protected AbstractDataset base; // is null when not a view
    049        protected int[] stride; // can be null for row-major, contiguous datasets
    050        protected int offset;
    051
    052        /**
    053         * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as
    054         * wanted
    055         */
    056        protected Serializable odata = null;
    057
    058        /**
    059         * Set aliased data as base data
    060         */
    061        abstract protected void setData();
    062
    063        /**
    064         * Constructor required for serialisation.
    065         */
    066        public AbstractDataset() {
    067        }
    068
    069        @Override
    070        public synchronized Dataset synchronizedCopy() {
    071                return clone();
    072        }
    073
    074        @Override
    075        public boolean equals(Object obj) {
    076                if (this == obj) {
    077                        return true;
    078                }
    079                if (obj == null) {
    080                        return false;
    081                }
    082                if (!getClass().equals(obj.getClass())) {
    083                        if (getRank() == 0) // for zero-rank datasets
    084                                return obj.equals(getObjectAbs(offset));
    085                        return false;
    086                }
    087
    088                Dataset other = (Dataset) obj;
    089                if (getElementsPerItem() != other.getElementsPerItem())
    090                        return false;
    091                if (size != other.getSize())
    092                        return false;
    093                if (!Arrays.equals(shape, other.getShapeRef())) {
    094                        return false;
    095                }
    096
    097                return true;
    098        }
    099
    100        @Override
    101        public int hashCode() {
    102                return getStats().getHash(shape);
    103        }
    104
    105        @Override
    106        abstract public AbstractDataset clone();
    107
    108        protected Format stringFormat = null;
    109
    110        @Override
    111        public void setStringFormat(Format format) {
    112                stringFormat = format;
    113        }
    114
    115        @Override
    116        public Dataset copy(final int dtype) {
    117                if (getDType() == dtype) {
    118                        return clone();
    119                }
    120                return DatasetUtils.copy(this, dtype);
    121        }
    122
    123        @Override
    124        public <T extends Dataset> T copy(Class<T> clazz) {
    125                return DatasetUtils.copy(clazz, this);
    126        }
    127
    128        @Override
    129        public Dataset cast(final int dtype) {
    130                if (getDType() == dtype) {
    131                        return this;
    132                }
    133                return DatasetUtils.cast(this, dtype);
    134        }
    135
    136        @Override
    137        public <T extends Dataset> T cast(Class<T> clazz) {
    138                return DatasetUtils.cast(clazz, this);
    139        }
    140
    141        @Override
    142        public Dataset cast(final boolean repeat, final int dtype, final int isize) {
    143                if (getDType() == dtype && getElementsPerItem() == isize) {
    144                        return this;
    145                }
    146                return DatasetUtils.cast(this, repeat, dtype, isize);
    147        }
    148
    149        @Override
    150        abstract public AbstractDataset getView(boolean deepCopyMetadata);
    151
    152        /**
    153         * Copy fields from original to view
    154         * @param orig
    155         * @param view
    156         * @param clone if true, then clone everything but bulk data
    157         * @param cloneMetadata if true, clone metadata
    158         */
    159        protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) {
    160                view.name = orig.getName();
    161                view.size = orig.getSize();
    162                view.odata = orig.getBuffer();
    163                view.offset = orig.getOffset();
    164                view.base = orig instanceof AbstractDataset ? ((AbstractDataset) orig).base : null;
    165
    166                if (clone) {
    167                        view.shape = orig.getShape();
    168                        view.stride = orig instanceof AbstractDataset && ((AbstractDataset) orig).stride != null ?
    169                                        ((AbstractDataset) orig).stride.clone() : null;
    170                } else {
    171                        view.shape = orig.getShapeRef();
    172                        view.stride = orig instanceof AbstractDataset ? ((AbstractDataset) orig).stride : null;
    173                }
    174
    175                view.metadata = getMetadataMap(orig, cloneMetadata);
    176                int odtype = orig.getDType();
    177                int vdtype = view.getDType();
    178                if (odtype != vdtype) {
    179                        view.setDirty();
    180                }
    181        }
    182
    183        /**
    184         * @since 2.0
    185         */
    186        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> getMetadataMap(Dataset a, boolean clone) {
    187                if (a == null)
    188                        return null;
    189
    190                List<MetadataType> all = null;
    191                try {
    192                        all = a.getMetadata(null);
    193                } catch (Exception e) {
    194                }
    195                if (all == null)
    196                        return null;
    197
    198                ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    199
    200                for (MetadataType m : all) {
    201                        if (m == null) {
    202                                continue;
    203                        }
    204                        Class<? extends MetadataType> c = findMetadataTypeSubInterfaces(m.getClass());
    205                        List<MetadataType> l = map.get(c);
    206                        if (l == null) {
    207                                l = new ArrayList<MetadataType>();
    208                                map.put(c, l);
    209                        }
    210                        if (clone)
    211                                m = m.clone();
    212                        l.add(m);
    213                }
    214                return map;
    215        }
    216
    217        @Override
    218        public IntegerDataset getIndices() {
    219                final IntegerDataset ret = DatasetUtils.indices(shape);
    220                if (getName() != null) {
    221                        ret.setName("Indices of " + getName());
    222                }
    223                return ret;
    224        }
    225
    226        @Override
    227        public Dataset getTransposedView(int... axes) {
    228                axes = checkPermutatedAxes(shape, axes);
    229
    230                AbstractDataset t = getView(true);
    231                if (axes == null || getRank() == 1)
    232                        return t;
    233
    234                int rank = shape.length;
    235                int[] tstride = new int[rank];
    236                int[] toffset = new int[1];
    237                int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset);
    238                int[] nstride = new int[rank];
    239                for (int i = 0; i < rank; i++) {
    240                        final int ax = axes[i];
    241                        nstride[i] = tstride[ax];
    242                        nshape[i] = shape[ax];
    243                }
    244                t.shape = nshape;
    245                t.stride = nstride;
    246                t.offset = toffset[0];
    247                t.base = this;
    248                t.setDirty();
    249                t.transposeMetadata(axes);
    250                return t;
    251        }
    252
    253        @Override
    254        public Dataset transpose(int... axes) {
    255                Dataset t = getTransposedView(axes);
    256                return t == null ? clone() : t.clone();
    257        }
    258
    259        @Override
    260        public Dataset swapAxes(int axis1, int axis2) {
    261                int rank = shape.length;
    262                axis1 = ShapeUtils.checkAxis(rank, axis1);
    263                axis2 = ShapeUtils.checkAxis(rank, axis2);
    264
    265                if (rank == 1 || axis1 == axis2) {
    266                        return this;
    267                }
    268
    269                int[] axes = new int[rank];
    270                for (int i = 0; i < rank; i++) {
    271                        axes[i] = i;
    272                }
    273
    274                axes[axis1] = axis2;
    275                axes[axis2] = axis1;
    276                return getTransposedView(axes);
    277        }
    278
    279        private boolean isContiguous() {
    280                if (stride == null)
    281                        return true;
    282
    283                if (offset != 0)
    284                        return false;
    285
    286                int s = getElementsPerItem();
    287                for (int j = getRank() - 1; j >= 0; j--) {
    288                        if (stride[j] != s) {
    289                                return false;
    290                        }
    291                        s *= shape[j];
    292                }
    293
    294                return true;
    295        }
    296
    297        @Override
    298        public Dataset flatten() {
    299                if (!isContiguous()) { // need to make a copy if not contiguous
    300                        return clone().flatten();
    301                }
    302                return reshape(size);
    303        }
    304
    305        /**
    306         * Fill dataset from object at depth dimension
    307         * @param obj
    308         * @param depth
    309         * @param pos position
    310         */
    311        protected void fillData(Object obj, final int depth, final int[] pos) {
    312                if (obj == null) {
    313                        int dtype = getDType();
    314                        if (dtype == FLOAT32)
    315                                set(Float.NaN, pos);
    316                        else if (dtype == FLOAT64)
    317                                set(Double.NaN, pos);
    318                        return;
    319                }
    320
    321                Class<?> clazz = obj.getClass();
    322                if (obj instanceof List<?>) {
    323                        List<?> jl = (List<?>) obj;
    324                        int l = jl.size();
    325                        for (int i = 0; i < l; i++) {
    326                                Object lo = jl.get(i);
    327                                fillData(lo, depth + 1, pos);
    328                                pos[depth]++;
    329                        }
    330                        pos[depth] = 0;
    331                } else if (clazz.isArray()) {
    332                        int l = Array.getLength(obj);
    333                        if (clazz.equals(odata.getClass())) {
    334                                System.arraycopy(obj, 0, odata, get1DIndex(pos), l);
    335                        } else if (clazz.getComponentType().isPrimitive()) {
    336                                for (int i = 0; i < l; i++) {
    337                                        set(Array.get(obj, i), pos);
    338                                        pos[depth]++;
    339                                }
    340                                pos[depth] = 0;
    341                        } else {
    342                                for (int i = 0; i < l; i++) {
    343                                        fillData(Array.get(obj, i), depth + 1, pos);
    344                                        pos[depth]++;
    345                                }
    346                                pos[depth] = 0;
    347                        }
    348                } else if (obj instanceof IDataset) {
    349                        boolean[] a = new boolean[shape.length];
    350                        for (int i = depth; i < a.length; i++)
    351                                a[i] = true;
    352                        setSlice(obj, getSliceIteratorFromAxes(pos, a));
    353                } else {
    354                        set(obj, pos);
    355                }
    356        }
    357
    358        @Override
    359        public IndexIterator getIterator(final boolean withPosition) {
    360                if (stride != null) {
    361                        return base.getSize() == 1  ? (withPosition ? new PositionIterator(offset, shape) :
    362                                new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset);
    363                }
    364                if (shape == null) {
    365                        return new NullIterator(shape, shape);
    366                }
    367
    368                return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size);
    369        }
    370
    371        @Override
    372        public IndexIterator getIterator() {
    373                return getIterator(false);
    374        }
    375
    376        @Override
    377        public PositionIterator getPositionIterator(final int... axes) {
    378                return new PositionIterator(shape, axes);
    379        }
    380
    381        @Override
    382        public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) {
    383                return getSliceIterator(new SliceND(shape, start, stop, step));
    384        }
    385
    386        /**
    387         * @param slice
    388         * @return an slice iterator that operates like an IndexIterator
    389         */
    390        public IndexIterator getSliceIterator(SliceND slice) {
    391                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
    392                        return new NullIterator(shape, slice.getShape());
    393                }
    394                if (stride != null) {
    395                        return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice);
    396                }
    397                return new SliceIterator(shape, size, slice);
    398        }
    399
    400        @Override
    401        public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) {
    402                int rank = shape.length;
    403                int[] start;
    404                int[] stop = new int[rank];
    405                int[] step = new int[rank];
    406
    407                if (pos == null) {
    408                        start = new int[rank];
    409                } else if (pos.length == rank) {
    410                        start = pos.clone();
    411                } else {
    412                        throw new IllegalArgumentException("pos array length is not equal to rank of dataset");
    413                }
    414                if (axes == null) {
    415                        axes = new boolean[rank];
    416                        Arrays.fill(axes, true);
    417                } else if (axes.length != rank) {
    418                        throw new IllegalArgumentException("axes array length is not equal to rank of dataset");
    419                }
    420
    421                for (int i = 0; i < rank; i++) {
    422                        if (axes[i]) {
    423                                stop[i] = shape[i];
    424                        } else {
    425                                stop[i] = start[i] + 1;
    426                        }
    427                        step[i] = 1;
    428                }
    429                return (SliceIterator) getSliceIterator(start, stop, step);
    430        }
    431
    432        @Override
    433        public BooleanIterator getBooleanIterator(Dataset choice) {
    434                return getBooleanIterator(choice, true);
    435        }
    436
    437        @Override
    438        public BooleanIterator getBooleanIterator(Dataset choice, boolean value) {
    439                return BooleanIterator.createIterator(value, this, choice, this);
    440        }
    441
    442        @Override
    443        public Dataset getByBoolean(Dataset selection) {
    444                checkCompatibility(selection);
    445
    446                final int length = ((Number) selection.sum()).intValue();
    447                final int is = getElementsPerItem();
    448                Dataset r = DatasetFactory.zeros(is, getClass(), length);
    449                BooleanIterator biter = getBooleanIterator(selection);
    450
    451                int i = 0;
    452                while (biter.hasNext()) {
    453                        r.setObjectAbs(i, getObjectAbs(biter.index));
    454                        i += is;
    455                }
    456                return r;
    457        }
    458
    459        @Override
    460        public Dataset getBy1DIndex(IntegerDataset index) {
    461                final int is = getElementsPerItem();
    462                final Dataset r = DatasetFactory.zeros(is, getClass(), index.getShape());
    463                final IntegerIterator iter = new IntegerIterator(index, size, is);
    464
    465                int i = 0;
    466                while (iter.hasNext()) {
    467                        r.setObjectAbs(i, getObjectAbs(iter.index));
    468                        i += is;
    469                }
    470                return r;
    471        }
    472
    473        @Override
    474        public Dataset getByIndexes(final Object... indexes) {
    475                final IntegersIterator iter = new IntegersIterator(shape, indexes);
    476                final int is = getElementsPerItem();
    477                final Dataset r = DatasetFactory.zeros(is, getClass(), iter.getShape());
    478
    479                final int[] pos = iter.getPos();
    480                int i = 0;
    481                while (iter.hasNext()) {
    482                        r.setObjectAbs(i, getObject(pos));
    483                        i += is;
    484                }
    485                return r;
    486        }
    487
    488        @Override
    489        public Class<?> getElementClass() {
    490                return DTypeUtils.getElementClass(getDType());
    491        }
    492
    493        @Override
    494        public boolean hasFloatingPointElements() {
    495                Class<?> cls = getElementClass();
    496                return cls == Float.class || cls == Double.class;
    497        }
    498
    499        @Override
    500        public int getElementsPerItem() {
    501                return DTypeUtils.getElementsPerItem(getDType());
    502        }
    503
    504        @Override
    505        public int getItemBytes() {
    506                return DTypeUtils.getItemBytes(getDType(), getElementsPerItem());
    507        }
    508
    509        @Override
    510        public String getName() {
    511                return name;
    512        }
    513
    514        @Override
    515        public void setName(final String name) {
    516                this.name = name;
    517        }
    518
    519        @Override
    520        public int getSize() {
    521                return size;
    522        }
    523
    524        @Override
    525        public int[] getShape() {
    526                // make a copy of the dimensions data, and put that out
    527                if (shape == null) {
    528                        logger.warn("Shape is null!!!");
    529                        return new int[] {};
    530                }
    531                return shape.clone();
    532        }
    533
    534        @Override
    535        public int getRank() {
    536                return shape == null ? 0 : shape.length;
    537        }
    538
    539        @Override
    540        public int getNbytes() {
    541                return getSize() * getItemBytes();
    542        }
    543
    544        /**
    545         * Check for -1 placeholder in shape and replace if necessary
    546         * @param shape
    547         * @param size
    548         */
    549        private void checkShape(int[] shape, int size) {
    550                int rank = shape.length;
    551                int found = -1;
    552                int nsize = 1;
    553                for (int i = 0; i < rank; i++) {
    554                        int d = shape[i];
    555                        if (d == -1) {
    556                                if (found == -1) {
    557                                        found = i;
    558                                } else {
    559                                        logger.error("Can only have one -1 placeholder in shape");
    560                                        throw new IllegalArgumentException("Can only have one -1 placeholder in shape");
    561                                }
    562                        } else {
    563                                nsize *= d;
    564                        }
    565                }
    566                if (found >= 0) {
    567                        shape[found] = size/nsize;
    568                } else if (nsize != size && !(rank == 0 && size == 0)) {
    569                        logger.error("New shape is not same size as old shape");
    570                        throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape));
    571                }
    572        }
    573
    574        @Override
    575        public void setShape(final int... shape) {
    576                int[] nshape = shape.clone();
    577                checkShape(nshape, size);
    578                if (Arrays.equals(this.shape, nshape)) {
    579                        return;
    580                }
    581
    582                if (stride != null) {
    583                        // the only compatible shapes are ones where new dimensions are factors of old dimensions
    584                        // or are combined adjacent old dimensions
    585                        int[] oshape = this.shape;
    586                        int orank = oshape.length;
    587                        int nrank = nshape.length;
    588                        int diff = nrank - orank;
    589                        int[] nstride = new int[nrank];
    590                        boolean ones = true;
    591                        // work forwards for broadcasting cases
    592                        for (int i = 0, j = 0; i < orank || j < nrank;) {
    593                                if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) {
    594                                        nstride[j++] = stride[i++];
    595                                } else if (j < nrank && nshape[j] == 1) {
    596                                        nstride[j++] = 0;
    597                                } else if (i < orank && oshape[i] == 1) {
    598                                        i++;
    599                                } else {
    600                                        if (j < nrank)
    601                                                j++;
    602                                        if (i < orank)
    603                                                i++;
    604                                        ones = false;
    605                                }
    606                        }
    607                        if (!ones) { // not just ones differ in shapes
    608                                int[] ostride = stride;
    609                                int ob = 0;
    610                                int oe = 1;
    611                                int nb = 0;
    612                                int ne = 1;
    613                                while (ob < orank && nb < nrank) {
    614                                        int ol = oshape[ob];
    615                                        int nl = nshape[nb];
    616
    617                                        if (nl < ol) { // find group of shape dimensions that form common size
    618                                                do { // case where new shape spreads single dimension over several dimensions
    619                                                        if (ne == nrank) {
    620                                                                break;
    621                                                        }
    622                                                        nl *= nshape[ne++];
    623                                                } while (nl < ol);
    624                                                if (nl != ol) {
    625                                                        logger.error("Subshape is incompatible with single dimension");
    626                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
    627                                                }
    628                                                int on = ne - 1;
    629                                                while (nshape[on] == 1) {
    630                                                        on--;
    631                                                }
    632
    633                                                nstride[on] = ostride[ob];
    634                                                for (int n = on - 1; n >= nb; n--) {
    635                                                        if (nshape[n] == 1)
    636                                                                continue;
    637
    638                                                        nstride[n] = nshape[on] * nstride[on];
    639                                                        on = n;
    640                                                }
    641                                        } else if (ol < nl) {
    642                                                do { // case where new shape combines several dimensions into one dimension
    643                                                        if (oe == orank) {
    644                                                                break;
    645                                                        }
    646                                                        ol *= oshape[oe++];
    647                                                } while (ol < nl);
    648                                                if (nl != ol) {
    649                                                        logger.error("Single dimension is incompatible with subshape");
    650                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
    651                                                }
    652
    653                                                int oo = oe - 1;
    654                                                while (oshape[oo] == 1) {
    655                                                        oo--;
    656                                                }
    657                                                int os = ostride[oo];
    658                                                for (int o = oo - 1; o >= ob; o--) {
    659                                                        if (oshape[o] == 1)
    660                                                                continue;
    661                                                        if (ostride[o] != oshape[oo] * ostride[oo]) {
    662                                                                logger.error("Subshape cannot be a non-contiguous view");
    663                                                                throw new IllegalArgumentException("Subshape cannot be a non-contiguous view");
    664                                                        }
    665                                                        oo = o;
    666                                                }
    667                                                nstride[nb] = os;
    668                                        } else {
    669                                                nstride[nb] = ostride[ob];
    670                                        }
    671
    672                                        ob = oe++;
    673                                        nb = ne++;
    674                                }
    675                        }
    676
    677                        stride = nstride;
    678                }
    679
    680                setDirty();
    681                if (this.shape != null) {
    682                        reshapeMetadata(this.shape, nshape);
    683                }
    684                this.shape = nshape;
    685        }
    686
    687        @Override
    688        public int[] getShapeRef() {
    689                return shape;
    690        }
    691
    692        @Override
    693        public int getOffset() {
    694                return offset;
    695        }
    696
    697        @Override
    698        public int[] getStrides() {
    699                return stride;
    700        }
    701
    702        @Override
    703        public Serializable getBuffer() {
    704                return odata;
    705        }
    706
    707        @Override
    708        public void overrideInternal(Serializable buffer, int... shape) {
    709                if (buffer != null) {
    710                        odata = buffer;
    711                        setData();
    712                        setDirty();
    713                }
    714
    715                if (shape != null) {
    716                        this.shape = shape.clone();
    717                        size = ShapeUtils.calcSize(this.shape);
    718                }
    719        }
    720
    721        /**
    722         * Create a stride array from dataset
    723         * @param a dataset
    724         * @param offset output offset
    725         * @return new strides
    726         */
    727        public static int[] createStrides(Dataset a, final int[] offset) {
    728                return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset);
    729        }
    730
    731        /**
    732         * Create a stride array from dataset
    733         * @param isize
    734         * @param shape
    735         * @param oStride original stride
    736         * @param oOffset original offset (only used if there is an original stride)
    737         * @param offset output offset
    738         * @return new strides
    739         */
    740        public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) {
    741                int rank = shape.length;
    742                final int[] stride;
    743                if (oStride == null) {
    744                        offset[0] = 0;
    745                        stride = new int[rank];
    746                        int s = isize;
    747                        for (int j = rank - 1; j >= 0; j--) {
    748                                stride[j] = s;
    749                                s *= shape[j];
    750                        }
    751                } else {
    752                        offset[0] = oOffset;
    753                        stride = oStride.clone();
    754                }
    755                return stride;
    756        }
    757
    758        /**
    759         * Create a stride array from slice information and a dataset
    760         * @param slice
    761         * @param a dataset
    762         * @param stride output stride
    763         * @param offset output offset
    764         * @return new shape
    765         */
    766        public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) {
    767                return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset);
    768        }
    769
    770        /**
    771         * Create a stride array from slice and dataset information
    772         * @param slice
    773         * @param isize
    774         * @param shape
    775         * @param oStride original stride
    776         * @param oOffset original offset (only used if there is an original stride)
    777         * @param stride output stride
    778         * @param offset output offset
    779         * @return new shape
    780         */
    781        public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) {
    782                int[] lstart = slice.getStart();
    783                int[] lstep = slice.getStep();
    784                int[] newShape = slice.getShape();
    785                int rank = shape.length;
    786
    787                if (oStride == null) {
    788                        int s = isize;
    789                        offset[0] = 0;
    790                        for (int j = rank - 1; j >= 0; j--) {
    791                                stride[j] = s * lstep[j];
    792                                offset[0] += s * lstart[j];
    793                                s *= shape[j];
    794                        }
    795                } else {
    796                        offset[0] = oOffset;
    797                        for (int j = 0; j < rank; j++) {
    798                                int s = oStride[j];
    799                                stride[j] = lstep[j] * s;
    800                                offset[0] += lstart[j] * s;
    801                        }
    802                }
    803
    804                return newShape;
    805        }
    806
    807        @Override
    808        public Dataset getBroadcastView(int... broadcastShape) {
    809                AbstractDataset view = getView(true);
    810
    811                if (!Arrays.equals(shape, broadcastShape)) {
    812                        List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape);
    813                        view.setShape(nShapes.get(0));
    814                        view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape);
    815                        view.base = this;
    816                        view.shape = broadcastShape.clone();
    817                        view.size = ShapeUtils.calcSize(broadcastShape);
    818                        if (view.name == null || view.name.isEmpty()) {
    819                                view.name = "Broadcast from " + Arrays.toString(shape);
    820                        } else {
    821                                view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape);
    822                        }
    823                }
    824                return view;
    825        }
    826
    827        @Override
    828        public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) {
    829                return getSliceView(new SliceND(shape, start, stop, step));
    830        }
    831
    832        @Override
    833        public Dataset getSliceView(Slice... slice) {
    834                if (slice == null || slice.length == 0) {
    835                        return getView(true);
    836                }
    837
    838                return getSliceView(new SliceND(shape, slice));
    839        }
    840
    841        /**
    842         * Get a slice of the dataset. The returned dataset is a view on a selection of items
    843         * @param slice
    844         * @return slice view
    845         */
    846        @Override
    847        public Dataset getSliceView(SliceND slice) {
    848                if (slice.isAll()) {
    849                        return getView(true);
    850                }
    851
    852                final int rank = shape.length;
    853                int[] sStride = new int[rank];
    854                int[] sOffset = new int[1];
    855
    856                int[] sShape = createStrides(slice, this, sStride, sOffset);
    857
    858                AbstractDataset s = getView(false);
    859                s.shape = sShape;
    860                s.size = ShapeUtils.calcSize(sShape);
    861                s.stride = sStride;
    862                s.offset = sOffset[0];
    863                s.base = this;
    864
    865                s.metadata = copyMetadata();
    866                s.sliceMetadata(true, slice);
    867
    868                s.setDirty();
    869                s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE);
    870
    871                return s;
    872        }
    873
    874        /**
    875         * Get flattened view index of given position
    876         * @param pos
    877         *            the integer array specifying the n-D position
    878         * @return the index on the flattened dataset
    879         */
    880        private int getFlat1DIndex(final int[] pos) {
    881                final int imax = pos.length;
    882                if (imax == 0) {
    883                        return 0;
    884                }
    885
    886                return get1DIndexFromShape(pos);
    887        }
    888
    889        /**
    890         * @since 2.0
    891         */
    892        protected int getFirst1DIndex() {
    893                if (shape == null) {
    894                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    895                }
    896                return stride == null ? 0 : offset;
    897        }
    898
    899        @Override
    900        public int get1DIndex(final int... n) {
    901                if (n.length == 0 && shape.length == 0)
    902                        return offset;
    903
    904                return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n);
    905        }
    906
    907        private static void throwAIOOBException(int i, int s, int d) {
    908                throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s
    909                                + "] in dimension " + d);
    910        }
    911
    912        /**
    913         * @param i
    914         * @return the index on the data array corresponding to that location
    915         */
    916        protected int get1DIndex(int i) {
    917                if (shape == null) {
    918                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    919                }
    920                if (shape.length > 1) {
    921                        logger.error("This dataset is not 1D but was addressed as such");
    922                        throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such");
    923                }
    924                if (i < 0) {
    925                        i += shape[0];
    926                }
    927                if (i < 0 || i >= shape[0]) {
    928                        throwAIOOBException(i, shape[0], 0);
    929                }
    930                return stride == null ? i : i*stride[0] + offset;
    931        }
    932
    933        /**
    934         * @param i
    935         * @param j
    936         * @return the index on the data array corresponding to that location
    937         */
    938        protected int get1DIndex(int i, int j) {
    939                if (shape == null) {
    940                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    941                }
    942                if (shape.length != 2) {
    943                        logger.error("This dataset is not 2D but was addressed as such");
    944                        throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such");
    945                }
    946                if (i < 0) {
    947                        i += shape[0];
    948                }
    949                if (i < 0 || i >= shape[0]) {
    950                        throwAIOOBException(i, shape[0], 0);
    951                }
    952                if (j < 0) {
    953                        j += shape[1];
    954                }
    955                if (j < 0 || j >= shape[1]) {
    956                        throwAIOOBException(i, shape[1], 1);
    957                }
    958                return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset;
    959        }
    960
    961        protected int get1DIndexFromShape(final int[] n) {
    962                return get1DIndexFromShape(shape, n);
    963        }
    964
    965        protected static int get1DIndexFromShape(final int[] shape, final int[] n) {
    966                if (shape == null) {
    967                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    968                }
    969                final int rank = shape.length;
    970                if (rank != n.length) {
    971                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
    972                        logger.error(errMsg);
    973                        throw new IllegalArgumentException(errMsg);
    974                }
    975                int index = 0;
    976                for (int i = 0; i < rank; i++) {
    977                        final int si = shape[i];
    978                        int ni = n[i];
    979                        if (ni < 0) {
    980                                ni += si;
    981                        }
    982                        if (ni < 0 || ni >= si) {
    983                                throwAIOOBException(ni, si, i);
    984                        }
    985                        index = index * si + ni;
    986                }
    987
    988                return index;
    989        }
    990
    991        private int get1DIndexFromStrides(final int[] n) {
    992                return get1DIndexFromStrides(shape, stride, offset, n);
    993        }
    994
    995        private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) {
    996                if (shape == null) {
    997                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    998                }
    999                final int rank = shape.length;
    1000                if (rank != n.length) {
    1001                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
    1002                        logger.error(errMsg);
    1003                        throw new IllegalArgumentException(errMsg);
    1004                }
    1005                int index = offset;
    1006                for (int i = 0; i < rank; i++) {
    1007                        final int st = stride[i];
    1008                        if (st != 0) { // not broadcasted
    1009                                final int si = shape[i];
    1010                                int ni = n[i];
    1011                                if (ni < 0) {
    1012                                        ni += si;
    1013                                }
    1014                                if (ni < 0 || ni >= si) {
    1015                                        throwAIOOBException(ni, si, i);
    1016                                }
    1017                                index += st * ni;
    1018                        }
    1019                }
    1020                return index;
    1021        }
    1022
    1023        @Override
    1024        public int[] getNDPosition(final int n) {
    1025                if (isIndexInRange(n)) {
    1026                        throw new IllegalArgumentException("Index provided " + n
    1027                                        + "is larger then the size of the containing array");
    1028                }
    1029
    1030                return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n);
    1031        }
    1032
    1033        private boolean isIndexInRange(final int n) {
    1034                if (stride == null) {
    1035                        return n >= size;
    1036                }
    1037                return n >= getBufferLength();
    1038        }
    1039
    1040        /**
    1041         * @return entire buffer length
    1042         */
    1043        abstract protected int getBufferLength();
    1044
    1045        private int[] getNDPositionFromStrides(int n) {
    1046                n -= offset;
    1047                int rank = shape.length;
    1048                if (rank == 1) {
    1049                        return new int[] { n / stride[0] };
    1050                }
    1051
    1052                int[] output = new int[rank];
    1053                int i = 0;
    1054                while (i != n) { // TODO find more efficient way than this exhaustive search
    1055                        int j = rank - 1;
    1056                        for (; j >= 0; j--) {
    1057                                output[j]++;
    1058                                i += stride[j];
    1059                                if (output[j] >= shape[j]) {
    1060                                        output[j] = 0;
    1061                                        i -= shape[j] * stride[j];
    1062                                } else {
    1063                                        break;
    1064                                }
    1065                        }
    1066                        if (j == -1) {
    1067                                logger.error("Index was not found in this strided dataset");
    1068                                throw new IllegalArgumentException("Index was not found in this strided dataset");
    1069                        }
    1070                }
    1071
    1072                return output;
    1073        }
    1074
    1075        @Override
    1076        public int checkAxis(int axis) {
    1077                return ShapeUtils.checkAxis(shape.length, axis);
    1078        }
    1079
    1080        @Deprecated
    1081        protected static int checkAxis(int rank, int axis) {
    1082                return ShapeUtils.checkAxis(rank, axis);
    1083        }
    1084
    1085        protected static final char BLOCK_OPEN = '[';
    1086        protected static final char BLOCK_CLOSE = ']';
    1087
    1088        @Override
    1089        public String toString() {
    1090                final int rank = shape == null ? 0 : shape.length;
    1091                final StringBuilder out = new StringBuilder();
    1092
    1093                if (DTypeUtils.isDTypeElemental(getDType())) {
    1094                        out.append("Dataset ");
    1095                } else {
    1096                        out.append("Compound dataset (");
    1097                        out.append(getElementsPerItem());
    1098                        out.append(") ");
    1099                }
    1100
    1101                if (name != null && name.length() > 0) {
    1102                        out.append("'");
    1103                        out.append(name);
    1104                        out.append("' has shape ");
    1105                } else {
    1106                        out.append("shape is ");
    1107                }
    1108
    1109                out.append(BLOCK_OPEN);
    1110                if (rank > 0) {
    1111                        out.append(shape[0]);
    1112                }
    1113                for (int i = 1; i < rank; i++) {
    1114                        out.append(", " + shape[i]);
    1115                }
    1116                out.append(BLOCK_CLOSE);
    1117                return out.toString();
    1118        }
    1119
    1120        @Override
    1121        public String toString(boolean showData) {
    1122                if (!showData) {
    1123                        return toString();
    1124                }
    1125
    1126                if (size == 0) {
    1127                        return "[]";
    1128                }
    1129
    1130                final int rank = shape == null ? 0 : shape.length;
    1131                final StringBuilder out = new StringBuilder();
    1132
    1133                if (rank > 0) {
    1134                        int[] pos = new int[rank];
    1135                        final StringBuilder lead = new StringBuilder();
    1136                        printBlocks(out, lead, 0, pos);
    1137                } else {
    1138                        out.append(getString());
    1139                }
    1140                return out.toString();
    1141        }
    1142
    1143        /**
    1144         * Limit to strings output via the toString() method
    1145         */
    1146        private static int maxStringLength = 120;
    1147
    1148        /**
    1149         * Set maximum line length for toString() method
    1150         * @param maxLineLength
    1151         */
    1152        public static void setMaxLineLength(int maxLineLength) {
    1153                maxStringLength = maxLineLength;
    1154        }
    1155
    1156        /**
    1157         * @return maximum line length for toString() method
    1158         */
    1159        public static int getMaxLineLength() {
    1160                return maxStringLength;
    1161        }
    1162
    1163        /**
    1164         * Limit to number of sub-blocks output via the toString() method
    1165         */
    1166        private static final int MAX_SUBBLOCKS = 6;
    1167
    1168        private final static String SEPARATOR = ",";
    1169        private final static String SPACE = " ";
    1170        private final static String ELLIPSIS = "...";
    1171        private final static String NEWLINE = "\n";
    1172
    1173        /**
    1174         * Make a line of output for last dimension of dataset
    1175         *
    1176         * @param start
    1177         * @return line
    1178         */
    1179        private StringBuilder makeLine(final int end, final int[] start) {
    1180                StringBuilder line = new StringBuilder();
    1181                final int[] pos;
    1182                if (end >= start.length) {
    1183                        pos = Arrays.copyOf(start, end + 1);
    1184                } else {
    1185                        pos = start;
    1186                }
    1187                pos[end] = 0;
    1188                line.append(BLOCK_OPEN);
    1189                line.append(getString(pos));
    1190
    1191                final int length = shape[end];
    1192
    1193                // trim elements printed if length exceed estimate of maximum elements
    1194                int excess = length - maxStringLength / 3; // space + number + separator
    1195                int midIndex = -1;
    1196                if (excess > 0) {
    1197                        int index = (length - excess) / 2;
    1198                        for (int y = 1; y < index; y++) {
    1199                                line.append(SEPARATOR + SPACE);
    1200                                pos[end] = y;
    1201                                line.append(getString(pos));
    1202                        }
    1203                        midIndex = line.length() + 2;
    1204                        index = (length + excess) / 2;
    1205                        for (int y = index; y < length; y++) {
    1206                                line.append(SEPARATOR + SPACE);
    1207                                pos[end] = y;
    1208                                line.append(getString(pos));
    1209                        }
    1210                } else {
    1211                        for (int y = 1; y < length; y++) {
    1212                                line.append(SEPARATOR + SPACE);
    1213                                pos[end] = y;
    1214                                line.append(getString(pos));
    1215                        }
    1216                }
    1217                line.append(BLOCK_CLOSE);
    1218
    1219                // trim string down to limit
    1220                int lineLength = line.length();
    1221                excess = lineLength - maxStringLength - ELLIPSIS.length() - 1;
    1222                if (excess > 0) {
    1223                        int index = (lineLength - excess) / 2;
    1224                        if (midIndex > 0 && index > midIndex) {
    1225                                index = midIndex;
    1226                        } else {
    1227                                index = line.lastIndexOf(SEPARATOR, index) + 2;
    1228                        }
    1229                        StringBuilder out = new StringBuilder(line.subSequence(0, index));
    1230                        out.append(ELLIPSIS + SEPARATOR);
    1231                        index = (lineLength + excess) / 2;
    1232                        if (midIndex > 0 && index <= midIndex) {
    1233                                index = midIndex - 1;
    1234                        } else {
    1235                                index = line.indexOf(SEPARATOR, index) + 1;
    1236                        }
    1237                        out.append(line.subSequence(index, lineLength));
    1238                        return out;
    1239                } else if (midIndex > 0) { // add ellipsis
    1240                        StringBuilder out = new StringBuilder(line.subSequence(0, midIndex));
    1241                        out.append(ELLIPSIS + SEPARATOR + SPACE);
    1242                        out.append(line.subSequence(midIndex, lineLength));
    1243                        return out;
    1244                }
    1245
    1246                return line;
    1247        }
    1248
    1249        /**
    1250         * recursive method to print blocks
    1251         */
    1252        private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) {
    1253                if (out.length() > 0) {
    1254                        char last = out.charAt(out.length() - 1);
    1255                        if (last != BLOCK_OPEN) {
    1256                                out.append(lead);
    1257                        }
    1258                }
    1259                final int end = getRank() - 1;
    1260                if (level != end) {
    1261                        out.append(BLOCK_OPEN);
    1262                        int length = shape[level];
    1263
    1264                        // first sub-block
    1265                        pos[level] = 0;
    1266                        StringBuilder newlead = new StringBuilder(lead);
    1267                        newlead.append(SPACE);
    1268                        printBlocks(out, newlead, level + 1, pos);
    1269                        if (length < 2) { // escape
    1270                                out.append(BLOCK_CLOSE);
    1271                                return;
    1272                        }
    1273
    1274                        out.append(SEPARATOR + NEWLINE);
    1275                        for (int i = level + 1; i < end; i++) {
    1276                                out.append(NEWLINE);
    1277                        }
    1278
    1279                        // middle sub-blocks
    1280                        if (length < MAX_SUBBLOCKS) {
    1281                                for (int x = 1; x < length - 1; x++) {
    1282                                        pos[level] = x;
    1283                                        printBlocks(out, newlead, level + 1, pos);
    1284                                        if (end <= level + 1) {
    1285                                                out.append(SEPARATOR + NEWLINE);
    1286                                        } else {
    1287                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
    1288                                        }
    1289                                }
    1290                        } else {
    1291                                final int excess = length - MAX_SUBBLOCKS;
    1292                                int xmax = (length - excess) / 2;
    1293                                for (int x = 1; x < xmax; x++) {
    1294                                        pos[level] = x;
    1295                                        printBlocks(out, newlead, level + 1, pos);
    1296                                        if (end <= level + 1) {
    1297                                                out.append(SEPARATOR + NEWLINE);
    1298                                        } else {
    1299                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
    1300                                        }
    1301                                }
    1302                                out.append(newlead);
    1303                                out.append(ELLIPSIS + SEPARATOR + NEWLINE);
    1304                                xmax = (length + excess) / 2;
    1305                                for (int x = xmax; x < length - 1; x++) {
    1306                                        pos[level] = x;
    1307                                        printBlocks(out, newlead, level + 1, pos);
    1308                                        if (end <= level + 1) {
    1309                                                out.append(SEPARATOR + NEWLINE);
    1310                                        } else {
    1311                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
    1312                                        }
    1313                                }
    1314                        }
    1315
    1316                        // last sub-block
    1317                        pos[level] = length - 1;
    1318                        printBlocks(out, newlead, level + 1, pos);
    1319                        out.append(BLOCK_CLOSE);
    1320                } else {
    1321                        out.append(makeLine(end, pos));
    1322                }
    1323        }
    1324
    1325        @Override
    1326        public Dataset squeezeEnds() {
    1327                return squeeze(true);
    1328        }
    1329
    1330        @Override
    1331        public Dataset squeeze() {
    1332                return squeeze(false);
    1333        }
    1334
    1335        @Override
    1336        public Dataset squeeze(boolean onlyFromEnds) {
    1337                final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds);
    1338                final int[] oshape = shape;
    1339                if (stride == null) {
    1340                        shape = tshape;
    1341                } else {
    1342                        int rank = shape.length;
    1343                        int trank = tshape.length;
    1344                        if (trank < rank) {
    1345                                int[] tstride = new int[tshape.length];
    1346                                if (onlyFromEnds) {
    1347                                        for (int i = 0; i < rank; i++) {
    1348                                                if (shape[i] != 1) {
    1349                                                        for (int k = 0; k < trank; k++) {
    1350                                                                tstride[k] = stride[i++];
    1351                                                        }
    1352                                                        break;
    1353                                                }
    1354                                        }
    1355                                } else {
    1356                                        int t = 0;
    1357                                        for (int i = 0; i < rank; i++) {
    1358                                                if (shape[i] != 1) {
    1359                                                        tstride[t++] = stride[i];
    1360                                                }
    1361                                        }
    1362                                }
    1363                                shape = tshape;
    1364                                stride = tstride;
    1365                        }
    1366                }
    1367
    1368                setDirty();
    1369                reshapeMetadata(oshape, shape);
    1370                return this;
    1371        }
    1372
    1373        @Override
    1374        public boolean isCompatibleWith(final ILazyDataset g) {
    1375                return ShapeUtils.areShapesCompatible(shape, g.getShape());
    1376        }
    1377
    1378        @Override
    1379        public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException {
    1380                ShapeUtils.checkCompatibility(this, g);
    1381        }
    1382
    1383        @Override
    1384        public Dataset reshape(final int... shape) {
    1385                Dataset a;
    1386                try {
    1387                        a = getView(true);
    1388                        a.setShape(shape);
    1389                } catch (IllegalArgumentException e) {
    1390                        a = clone();
    1391                        a.setShape(shape);
    1392                }
    1393                return a;
    1394        }
    1395
    1396        /**
    1397         * @param start
    1398         * @param stop
    1399         * @param step
    1400         * @return number of steps to take
    1401         */
    1402        protected static int calcSteps(final double start, final double stop, final double step) {
    1403                return Math.max(0, (int) Math.ceil((stop - start) / step));
    1404        }
    1405
    1406        @Override
    1407        public boolean isComplex() {
    1408                int type = getDType();
    1409                return type == COMPLEX64 || type == COMPLEX128;
    1410        }
    1411
    1412        @Override
    1413        public Dataset getRealPart() {
    1414                return this;
    1415        }
    1416
    1417        @Override
    1418        public Dataset getRealView() {
    1419                return getView(true);
    1420        }
    1421
    1422        @Override
    1423        public Dataset getSlice(final int[] start, final int[] stop, final int[] step) {
    1424                return getSlice(new SliceND(shape, start, stop, step));
    1425        }
    1426
    1427        @Override
    1428        public Dataset getSlice(Slice... slice) {
    1429                return getSlice(new SliceND(shape, slice));
    1430        }
    1431
    1432        @Override
    1433        public Dataset getSlice(IMonitor monitor, Slice... slice) {
    1434                return getSlice(slice);
    1435        }
    1436
    1437        @Override
    1438        public Dataset getSlice(IMonitor monitor, SliceND slice) {
    1439                return getSlice(slice);
    1440        }
    1441
    1442        @Override
    1443        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) {
    1444                return getSlice(start, stop, step);
    1445        }
    1446
    1447        /**
    1448         * Get a slice of the dataset. The returned dataset is a copied selection of items
    1449         * @param slice
    1450         * @return The dataset of the sliced data
    1451         */
    1452        @Override
    1453        public Dataset getSlice(final SliceND slice) {
    1454                SliceIterator it = (SliceIterator) getSliceIterator(slice);
    1455                AbstractDataset s = getSlice(it);
    1456                s.metadata = copyMetadata();
    1457                s.setDirty();
    1458                s.sliceMetadata(true, slice);
    1459                return s;
    1460        }
    1461
    1462        /**
    1463         * Get a slice of the dataset. The returned dataset is a copied selection of items
    1464         *
    1465         * @param iterator Slice iterator
    1466         * @return The dataset of the sliced data
    1467         */
    1468        abstract public AbstractDataset getSlice(final SliceIterator iterator);
    1469
    1470        @SuppressWarnings("deprecation")
    1471        @Override
    1472        public Dataset setSlice(final Object obj, final SliceND slice) {
    1473                Dataset ds;
    1474                if (obj instanceof Dataset) {
    1475                        ds = (Dataset) obj;
    1476                } else if (obj instanceof IDataset) {
    1477                        ds = DatasetUtils.convertToDataset((IDataset) obj);
    1478                } else {
    1479                        int dtype = getDType();
    1480                        if (dtype != Dataset.BOOL) {
    1481                                dtype = DTypeUtils.getLargestDType(dtype);
    1482                        }
    1483                        ds = DatasetFactory.createFromObject(getElementsPerItem(), dtype, obj);
    1484                }
    1485
    1486                return setSlicedView(getSliceView(slice), ds);
    1487        }
    1488
    1489        @Override
    1490        public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) {
    1491                return setSlice(obj, new SliceND(shape, start, stop, step));
    1492        }
    1493
    1494        /**
    1495         * Set a view of current dataset to given dataset with broadcasting
    1496         * @param view
    1497         * @param d
    1498         * @return this dataset
    1499         */
    1500        abstract Dataset setSlicedView(Dataset view, Dataset d);
    1501
    1502        @Override
    1503        public Dataset setSlice(Object obj, Slice... slice) {
    1504                if (slice == null || slice.length == 0) {
    1505                        return setSlice(obj, new SliceND(shape));
    1506                }
    1507                return setSlice(obj, new SliceND(shape, slice));
    1508        }
    1509
    1510        @Override
    1511        public boolean all() {
    1512                return Comparisons.allTrue(this);
    1513        }
    1514
    1515        @Override
    1516        public BooleanDataset all(final int axis) {
    1517                return Comparisons.allTrue(this, axis);
    1518        }
    1519
    1520        @Override
    1521        public boolean any() {
    1522                return Comparisons.anyTrue(this);
    1523        }
    1524
    1525        @Override
    1526        public BooleanDataset any(final int axis) {
    1527                return Comparisons.anyTrue(this, axis);
    1528        }
    1529
    1530        @Override
    1531        public Dataset ifloorDivide(final Object o) {
    1532                return idivide(o).ifloor();
    1533        }
    1534
    1535        @Override
    1536        public double residual(final Object o) {
    1537                return residual(o, null, false);
    1538        }
    1539
    1540        @Override
    1541        public double residual(final Object o, boolean ignoreNaNs) {
    1542                return residual(o, null, ignoreNaNs);
    1543        }
    1544
    1545        /**
    1546         * @since 2.0
    1547         */
    1548        @SuppressWarnings("unchecked")
    1549        protected StatisticsMetadata<Number> getStats() {
    1550                StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class);
    1551                if (md == null || md.isDirty()) {
    1552                        md = new StatisticsMetadataImpl<Number>();
    1553                        md.initialize(this);
    1554                        setMetadata(md);
    1555                }
    1556                return md;
    1557        }
    1558
    1559        /**
    1560         * @since 2.0
    1561         */
    1562        @SuppressWarnings("unchecked")
    1563        protected StatisticsMetadata<String> getStringStats() {
    1564                StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class);
    1565                if (md == null || md.isDirty()) {
    1566                        md = new StatisticsMetadataImpl<String>();
    1567                        md.initialize(this);
    1568                        setMetadata(md);
    1569                }
    1570                return md;
    1571        }
    1572
    1573        @Override
    1574        public Number max(boolean... ignoreInvalids) {
    1575                return getStats().getMaximum(ignoreInvalids);
    1576        }
    1577
    1578        @Override
    1579        public Dataset max(int axis, boolean... ignoreInvalids) {
    1580                return getStats().getMaximum(axis, ignoreInvalids);
    1581        }
    1582
    1583        @Override
    1584        public Number min(boolean... ignoreInvalids) {
    1585                return getStats().getMinimum(ignoreInvalids);
    1586        }
    1587
    1588        @Override
    1589        public Dataset min(int axis, boolean... ignoreInvalids) {
    1590                return getStats().getMinimum(axis, ignoreInvalids);
    1591        }
    1592
    1593        @Override
    1594        public int argMax(boolean... ignoreInvalids) {
    1595                return getFlat1DIndex(maxPos(ignoreInvalids));
    1596        }
    1597
    1598        /**
    1599         * @since 2.0
    1600         */
    1601        @Override
    1602        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
    1603                return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids);
    1604        }
    1605
    1606        @Override
    1607        public int argMin(boolean... ignoreInvalids) {
    1608                return getFlat1DIndex(minPos(ignoreInvalids));
    1609        }
    1610
    1611        /**
    1612         * @since 2.0
    1613         */
    1614        @Override
    1615        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
    1616                return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids);
    1617        }
    1618
    1619        @Override
    1620        public Number peakToPeak(boolean... ignoreInvalids) {
    1621                return DTypeUtils.fromDoubleToBiggestNumber(max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue(), getDType());
    1622        }
    1623
    1624        @Override
    1625        public Dataset peakToPeak(int axis,  boolean... ignoreInvalids) {
    1626                return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids));
    1627        }
    1628
    1629
    1630        @Override
    1631        public long count(boolean... ignoreInvalids) {
    1632                return getStats().getCount(ignoreInvalids);
    1633        }
    1634
    1635        @Override
    1636        public Dataset count(int axis, boolean... ignoreInvalids) {
    1637                return getStats().getCount(axis, ignoreInvalids);
    1638        }
    1639
    1640        @Override
    1641        public Object sum(boolean... ignoreInvalids) {
    1642                return getStats().getSum(ignoreInvalids);
    1643        }
    1644
    1645        @Override
    1646        public Dataset sum(int axis, boolean... ignoreInvalids) {
    1647                return getStats().getSum(axis, ignoreInvalids);
    1648        }
    1649
    1650        @Override
    1651        public Object product(boolean... ignoreInvalids) {
    1652                return Stats.product(this, ignoreInvalids);
    1653        }
    1654
    1655        @Override
    1656        public Dataset product(int axis, boolean... ignoreInvalids) {
    1657                return Stats.product(this, axis, ignoreInvalids);
    1658        }
    1659
    1660        @Override
    1661        public Object mean(boolean... ignoreInvalids) {
    1662                return getStats().getMean(ignoreInvalids);
    1663        }
    1664
    1665        @Override
    1666        public Dataset mean(int axis, boolean... ignoreInvalids) {
    1667                return getStats().getMean(axis, ignoreInvalids);
    1668        }
    1669
    1670        @Override
    1671        public double variance() {
    1672                return variance(false);
    1673        }
    1674
    1675        @Override
    1676        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
    1677                return getStats().getVariance(isWholePopulation, ignoreInvalids);
    1678        }
    1679
    1680        @Override
    1681        public Dataset variance(int axis) {
    1682                return getStats().getVariance(axis, false);
    1683        }
    1684
    1685        @Override
    1686        public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    1687                return getStats().getVariance(axis, isWholePopulation, ignoreInvalids);
    1688        }
    1689
    1690        @Override
    1691        public double stdDeviation() {
    1692                return Math.sqrt(variance());
    1693        }
    1694
    1695        @Override
    1696        public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) {
    1697                return Math.sqrt(variance(isWholePopulation, ignoreInvalids));
    1698        }
    1699
    1700        @Override
    1701        public Dataset stdDeviation(int axis) {
    1702                return Maths.sqrt(variance(axis, false));
    1703        }
    1704
    1705        @Override
    1706        public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    1707                return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids));
    1708        }
    1709
    1710        @Override
    1711        public double rootMeanSquare(boolean... ignoreInvalids) {
    1712                StatisticsMetadata<Number> stats = getStats();
    1713                final double mean = stats.getMean(ignoreInvalids).doubleValue();
    1714                final double var = stats.getVariance(true, ignoreInvalids);
    1715                return Math.sqrt(var + mean * mean);
    1716        }
    1717
    1718        @Override
    1719        public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
    1720                StatisticsMetadata<Number> stats = getStats();
    1721                Dataset v = stats.getVariance(axis, true, ignoreInvalids);
    1722                Dataset m = stats.getMean(axis, ignoreInvalids);
    1723                Dataset result = Maths.multiply(m, m);
    1724                return Maths.sqrt(result.iadd(v));
    1725        }
    1726
    1727        /**
    1728         * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards.
    1729         *
    1730         * @param dindex
    1731         * @param sindex
    1732         * @param src
    1733         *            is the source data buffer
    1734         */
    1735        protected abstract void setItemDirect(final int dindex, final int sindex, final Object src);
    1736
    1737        /**
    1738         * @return error broadcasted to current shape
    1739         */
    1740        private Dataset getBroadcastedInternalError() {
    1741                if (shape == null) {
    1742                        throw new IllegalArgumentException("Cannot get error for null dataset");
    1743                }
    1744                ILazyDataset led = super.getErrors();
    1745                if (led == null)
    1746                        return null;
    1747
    1748                Dataset ed = null;
    1749                try {
    1750                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
    1751                } catch (DatasetException e) {
    1752                        logger.error("Could not get data from lazy dataset", e);
    1753                }
    1754                if (led != ed) {
    1755                        setErrors(ed); // set back
    1756                }
    1757
    1758                return ed.getBroadcastView(shape);
    1759        }
    1760
    1761        @Override
    1762        public Dataset getErrors() {
    1763                Dataset ed = getBroadcastedInternalError();
    1764                if (ed == null)
    1765                        return null;
    1766
    1767                return ed;
    1768        }
    1769
    1770        @Override
    1771        public double getError() {
    1772                Dataset ed = getBroadcastedInternalError();
    1773                if (ed == null)
    1774                        return 0;
    1775
    1776                return ed.getDouble();
    1777        }
    1778
    1779        @Override
    1780        public double getError(final int i) {
    1781                Dataset ed = getBroadcastedInternalError();
    1782                if (ed == null)
    1783                        return 0;
    1784
    1785                return ed.getDouble(i);
    1786        }
    1787
    1788        @Override
    1789        public double getError(final int i, final int j) {
    1790                Dataset ed = getBroadcastedInternalError();
    1791                if (ed == null)
    1792                        return 0;
    1793
    1794                return ed.getDouble(i, j);
    1795        }
    1796
    1797        @Override
    1798        public double getError(int... pos) {
    1799                Dataset ed = getBroadcastedInternalError();
    1800                if (ed == null)
    1801                        return 0;
    1802
    1803                return ed.getDouble(pos);
    1804        }
    1805
    1806        @Override
    1807        public double[] getErrorArray(final int i) {
    1808                Dataset ed = getBroadcastedInternalError();
    1809                if (ed == null)
    1810                        return null;
    1811
    1812                return new double[] {getError(i)};
    1813        }
    1814
    1815        @Override
    1816        public double[] getErrorArray(final int i, final int j) {
    1817                Dataset ed = getBroadcastedInternalError();
    1818                if (ed == null)
    1819                        return null;
    1820
    1821                return new double[] {getError(i, j)};
    1822        }
    1823
    1824        @Override
    1825        public double[] getErrorArray(int... pos) {
    1826                Dataset ed = getBroadcastedInternalError();
    1827                if (ed == null)
    1828                        return null;
    1829
    1830                return new double[] {getError(pos)};
    1831        }
    1832
    1833        protected Dataset getInternalSquaredError() {
    1834                Dataset sed = getErrorBuffer().getBroadcastView(shape);
    1835                return sed;
    1836        }
    1837
    1838        @Override
    1839        public Dataset getErrorBuffer() {
    1840                ErrorMetadata emd = getErrorMetadata();
    1841                if (emd == null)
    1842                        return null;
    1843
    1844                if (!(emd instanceof ErrorMetadataImpl)) {
    1845                        ILazyDataset led = emd.getError();
    1846                        Dataset ed;
    1847                        try {
    1848                                ed = DatasetUtils.sliceAndConvertLazyDataset(led);
    1849                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1850                                setMetadata(emd);
    1851                                emd.setError(ed);
    1852                        } catch (MetadataException me) {
    1853                                logger.error("Could not create metadata", me);
    1854                        } catch (DatasetException e) {
    1855                                logger.error("Could not get data from lazy dataset", e);
    1856                        }
    1857                }
    1858
    1859                return ((ErrorMetadataImpl) emd).getSquaredError();
    1860        }
    1861
    1862        /**
    1863         * Set a copy of the buffer that backs the (squared) error data
    1864         * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset
    1865         */
    1866        @Override
    1867        public void setErrorBuffer(Serializable buffer) {
    1868                if (shape == null) {
    1869                        throw new IllegalArgumentException("Cannot set error buffer for null dataset");
    1870                }
    1871                if (buffer == null) {
    1872                        clearMetadata(ErrorMetadata.class);
    1873                        return;
    1874                }
    1875
    1876                IDataset d = (IDataset) createFromSerializable(buffer, false);
    1877                ErrorMetadata emd = getErrorMetadata();
    1878                if (!(emd instanceof ErrorMetadataImpl)) {
    1879                        try {
    1880                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1881                                setMetadata(emd);
    1882                        } catch (MetadataException me) {
    1883                                logger.error("Could not create metadata", me);
    1884                        }
    1885                }
    1886                ((ErrorMetadataImpl) emd).setSquaredError(d);
    1887        }
    1888}