    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.Serializable;
    016import java.lang.reflect.Array;
    017import java.text.Format;
    018import java.util.Arrays;
    019import java.util.List;
    020
    021import org.eclipse.january.DatasetException;
    022import org.eclipse.january.IMonitor;
    023import org.eclipse.january.MetadataException;
    024import org.eclipse.january.metadata.ErrorMetadata;
    025import org.eclipse.january.metadata.MetadataFactory;
    026import org.eclipse.january.metadata.StatisticsMetadata;
    027import org.eclipse.january.metadata.internal.ErrorMetadataImpl;
    028import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
    029import org.slf4j.Logger;
    030import org.slf4j.LoggerFactory;
    031
    032/**
    033 * Generic container class for data
    034 * <p>
    035 * Each subclass has an array of primitive types, elements of this array are grouped or
    036 * compounded to make items
    037 * <p>
    038 * Data items can be boolean, integer, float, complex float, vector float, etc
    039 */
    040public abstract class AbstractDataset extends LazyDatasetBase implements Dataset {
    041        // pin UID to base class
    042        private static final long serialVersionUID = Dataset.serialVersionUID;
    043
    044        private static final Logger logger = LoggerFactory.getLogger(AbstractDataset.class);
    045
    046        protected int size; // number of items
    047
    048        transient protected AbstractDataset base; // is null when not a view
    049        protected int[] stride; // can be null for row-major, contiguous datasets
    050        protected int offset;
    051
    052        /**
    053         * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as
    054         * wanted
    055         */
    056        protected Serializable odata = null;
    057
    058        /**
    059         * Set aliased data as base data
    060         */
    061        abstract protected void setData();
    062
    063        /**
    064         * Constructor required for serialisation.
    065         */
    066        public AbstractDataset() {
    067        }
    068
    069        @Override
    070        public synchronized Dataset synchronizedCopy() {
    071                return clone();
    072        }
    073
    074        @Override
    075        public Class<?> getElementClass() {
    076                return InterfaceUtils.getElementClass(getClass());
    077        }
    078
    079        @Override
    080        public int getDType() {
    081                return DTypeUtils.getDType(getClass());
    082        }
    083
    084        @Override
    085        public int hashCode() {
    086                return getStats().getHash(shape);
    087        }
    088
    089        @Override
    090        abstract public AbstractDataset clone();
    091
    092        protected Format stringFormat = null;
    093
    094        @Override
    095        public void setStringFormat(Format format) {
    096                stringFormat = format;
    097        }
    098
    099        @Override
    100        public Dataset copy(final int dtype) {
    101                return copy(DTypeUtils.getInterface(dtype));
    102        }
    103
    104        @Override
    105        public <T extends Dataset> T copy(Class<T> clazz) {
    106                return DatasetUtils.copy(clazz, this);
    107        }
    108
    109        @Override
    110        public Dataset cast(final int dtype) {
    111                return cast(DTypeUtils.getInterface(dtype));
    112        }
    113
    114        @SuppressWarnings("unchecked")
    115        @Override
    116        public <T extends Dataset> T cast(Class<T> clazz) {
    117                if (clazz.isInstance(this)) {
    118                        return (T) this;
    119                }
    120                return DatasetUtils.cast(clazz, this);
    121        }
    122
    123        @SuppressWarnings("unchecked")
    124        @Override
    125        public <T extends Dataset> T cast(final int isize, Class<T> clazz, final boolean repeat) {
    126                if (clazz.isInstance(this) && getElementsPerItem() == isize) {
    127                        return (T) this;
    128                }
    129                return DatasetUtils.cast(isize, clazz, this, repeat);
    130        }
    131
    132        @Override
    133        public Dataset cast(final boolean repeat, final int dtype, final int isize) {
    134                return cast(isize, DTypeUtils.getInterface(dtype), repeat);
    135        }
    136
    137        @Override
    138        abstract public AbstractDataset getView(boolean deepCopyMetadata);
    139
    140        /**
    141         * Copy fields from original to view
    142         * @param orig original
    143         * @param view destination
    144         * @param clone if true, then clone everything but bulk data
    145         * @param cloneMetadata if true, clone metadata
    146         */
    147        protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) {
    148                view.name = orig.getName();
    149                view.size = orig.getSize();
    150                view.odata = orig.getBuffer();
    151                view.offset = orig.getOffset();
    152                AbstractDataset a = orig instanceof AbstractDataset ? (AbstractDataset) orig : null;
    153                view.base = a == null ? null : a.base;
    154
    155                int[] s = a == null ? null : a.stride;
    156                if (clone) {
    157                        view.shape = orig.getShapeRef() == null ? null : orig.getShape();
    158                        view.stride = s == null ? null : s.clone();
    159                } else {
    160                        view.shape = orig.getShapeRef();
    161                        view.stride = s;
    162                }
    163
    164                view.metadata = getMetadataMap(orig, cloneMetadata);
    165                Class<? extends Dataset> oc = InterfaceUtils.findSubInterface(orig.getClass());
    166                Class<? extends Dataset> vc = InterfaceUtils.findSubInterface(view.getClass());
    167                if (!oc.equals(vc)) {
    168                        view.setDirty();
    169                }
    170        }
    171
    172        @Override
    173        public IntegerDataset getIndices() {
    174                final IntegerDataset ret = DatasetUtils.indices(shape);
    175                if (getName() != null) {
    176                        ret.setName("Indices of " + getName());
    177                }
    178                return ret;
    179        }
    180
    181        @Override
    182        public Dataset getTransposedView(int... axes) {
    183                axes = checkPermutatedAxes(shape, axes);
    184
    185                AbstractDataset t = getView(true);
    186                if (axes == null || getRank() == 1)
    187                        return t;
    188
    189                int rank = shape.length;
    190                int[] tstride = new int[rank];
    191                int[] toffset = new int[1];
    192                int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset);
    193                int[] nstride = new int[rank];
    194                for (int i = 0; i < rank; i++) {
    195                        final int ax = axes[i];
    196                        nstride[i] = tstride[ax];
    197                        nshape[i] = shape[ax];
    198                }
    199                t.shape = nshape;
    200                t.stride = nstride;
    201                t.offset = toffset[0];
    202                t.base = this;
    203                t.setDirty();
    204                t.transposeMetadata(axes);
    205                return t;
    206        }
    207
    208        @Override
    209        public Dataset transpose(int... axes) {
    210                Dataset t = getTransposedView(axes);
    211                return t == null ? clone() : t.clone();
    212        }
    213
    214        @Override
    215        public Dataset swapAxes(int axis1, int axis2) {
    216                int rank = shape.length;
    217                axis1 = ShapeUtils.checkAxis(rank, axis1);
    218                axis2 = ShapeUtils.checkAxis(rank, axis2);
    219
    220                if (rank == 1 || axis1 == axis2) {
    221                        return this;
    222                }
    223
    224                int[] axes = new int[rank];
    225                for (int i = 0; i < rank; i++) {
    226                        axes[i] = i;
    227                }
    228
    229                axes[axis1] = axis2;
    230                axes[axis2] = axis1;
    231                return getTransposedView(axes);
    232        }
    233
    234        boolean isContiguous() {
    235                if (stride == null)
    236                        return true;
    237
    238                if (offset != 0)
    239                        return false;
    240
    241                int s = getElementsPerItem();
    242                for (int j = getRank() - 1; j >= 0; j--) {
    243                        if (stride[j] != s) {
    244                                return false;
    245                        }
    246                        s *= shape[j];
    247                }
    248
    249                return true;
    250        }
    251
    252        @Override
    253        public Dataset flatten() {
    254                if (!isContiguous()) { // need to make a copy if not contiguous
    255                        return clone().flatten();
    256                }
    257                return reshape(size);
    258        }
    259
    260        /**
    261         * Fill dataset from object at depth dimension
    262         * @param obj fill value
    263         * @param depth dimension
    264         * @param pos position
    265         */
    266        protected void fillData(Object obj, final int depth, final int[] pos) {
    267                if (obj == null) {
    268                        Class<?> c = InterfaceUtils.getElementClass(getClass());
    269                        if (Float.class.equals(c)) {
    270                                set(Float.NaN, pos);
    271                        } else if (Float.class.equals(c)) {
    272                                set(Double.NaN, pos);
    273                        }
    274                        return;
    275                }
    276
    277                Class<?> clazz = obj.getClass();
    278                if (obj instanceof List<?>) {
    279                        List<?> jl = (List<?>) obj;
    280                        int l = jl.size();
    281                        for (int i = 0; i < l; i++) {
    282                                Object lo = jl.get(i);
    283                                fillData(lo, depth + 1, pos);
    284                                pos[depth]++;
    285                        }
    286                        pos[depth] = 0;
    287                } else if (clazz.isArray()) {
    288                        int l = Array.getLength(obj);
    289                        if (clazz.equals(odata.getClass())) {
    290                                System.arraycopy(obj, 0, odata, get1DIndex(pos), l);
    291                        } else if (clazz.getComponentType().isPrimitive()) {
    292                                for (int i = 0; i < l; i++) {
    293                                        set(Array.get(obj, i), pos);
    294                                        pos[depth]++;
    295                                }
    296                                pos[depth] = 0;
    297                        } else {
    298                                for (int i = 0; i < l; i++) {
    299                                        fillData(Array.get(obj, i), depth + 1, pos);
    300                                        pos[depth]++;
    301                                }
    302                                pos[depth] = 0;
    303                        }
    304                } else if (obj instanceof IDataset) {
    305                        boolean[] a = new boolean[shape.length];
    306                        Arrays.fill(a, depth, a.length, true);
    307                        setSlice(obj, getSliceIteratorFromAxes(pos, a));
    308                } else {
    309                        set(obj, pos);
    310                }
    311        }
    312
    313        @Override
    314        public IndexIterator getIterator(final boolean withPosition) {
    315                if (stride != null) {
    316                        return base.getSize() == 1  ? (withPosition ? new PositionIterator(offset, shape) :
    317                                new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset);
    318                }
    319                if (shape == null) {
    320                        return new NullIterator();
    321                }
    322
    323                return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size);
    324        }
    325
    326        @Override
    327        public IndexIterator getIterator() {
    328                return getIterator(false);
    329        }
    330
    331        @Override
    332        public PositionIterator getPositionIterator(final int... axes) {
    333                return new PositionIterator(shape, axes);
    334        }
    335
    336        @Override
    337        public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) {
    338                return internalGetSliceIterator(new SliceND(shape, start, stop, step));
    339        }
    340
    341        /**
    342         * @param slice an n-D slice
    343         * @return an slice iterator that operates like an IndexIterator
    344         */
    345        public IndexIterator getSliceIterator(SliceND slice) {
    346                if (slice != null) {
    347                        checkSliceND(slice);
    348                }
    349                return internalGetSliceIterator(slice);
    350        }
    351
    352        /**
    353         * @param slice an n-D slice
    354         * @return an slice iterator that operates like an IndexIterator
    355         */
    356        protected IndexIterator internalGetSliceIterator(SliceND slice) {
    357                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
    358                        return new NullIterator(shape, slice.getShape());
    359                }
    360                if (stride != null) {
    361                        return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice);
    362                }
    363                return new SliceIterator(shape, size, slice);
    364        }
    365
    366        @Override
    367        public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) {
    368                int rank = shape.length;
    369                int[] start;
    370                int[] stop = new int[rank];
    371                int[] step = new int[rank];
    372
    373                if (pos == null) {
    374                        start = new int[rank];
    375                } else if (pos.length == rank) {
    376                        start = pos.clone();
    377                } else {
    378                        throw new IllegalArgumentException("pos array length is not equal to rank of dataset");
    379                }
    380                if (axes == null) {
    381                        axes = new boolean[rank];
    382                        Arrays.fill(axes, true);
    383                } else if (axes.length != rank) {
    384                        throw new IllegalArgumentException("axes array length is not equal to rank of dataset");
    385                }
    386
    387                for (int i = 0; i < rank; i++) {
    388                        if (axes[i]) {
    389                                stop[i] = shape[i];
    390                        } else {
    391                                stop[i] = start[i] + 1;
    392                        }
    393                        step[i] = 1;
    394                }
    395                return (SliceIterator) getSliceIterator(start, stop, step);
    396        }
    397
    398        @Override
    399        public BooleanIterator getBooleanIterator(Dataset choice) {
    400                return getBooleanIterator(choice, true);
    401        }
    402
    403        @Override
    404        public BooleanIterator getBooleanIterator(Dataset choice, boolean value) {
    405                return BooleanIterator.createIterator(value, this, choice, this);
    406        }
    407
    408        @Override
    409        public Dataset getByBoolean(Dataset selection) {
    410                checkCompatibility(selection);
    411
    412                final int length = ((Number) selection.sum()).intValue();
    413                final int is = getElementsPerItem();
    414                Dataset r = DatasetFactory.zeros(is, getClass(), length);
    415                BooleanIterator biter = getBooleanIterator(selection);
    416
    417                int i = 0;
    418                while (biter.hasNext()) {
    419                        r.setObjectAbs(i, getObjectAbs(biter.index));
    420                        i += is;
    421                }
    422                return r;
    423        }
    424
    425        @Override
    426        public Dataset getBy1DIndex(IntegerDataset index) {
    427                final int is = getElementsPerItem();
    428                final Dataset r = DatasetFactory.zeros(is, getClass(), index.getShapeRef());
    429                final IntegerIterator iter = new IntegerIterator(index, size, is);
    430
    431                int i = 0;
    432                while (iter.hasNext()) {
    433                        r.setObjectAbs(i, getObjectAbs(iter.index));
    434                        i += is;
    435                }
    436                return r;
    437        }
    438
    439        @Override
    440        public Dataset getByIndexes(final Object... indexes) {
    441                final IntegersIterator iter = new IntegersIterator(shape, indexes);
    442                final int is = getElementsPerItem();
    443                final Dataset r = DatasetFactory.zeros(is, getClass(), iter.getShape());
    444
    445                final int[] pos = iter.getPos();
    446                int i = 0;
    447                while (iter.hasNext()) {
    448                        r.setObjectAbs(i, getObject(pos));
    449                        i += is;
    450                }
    451                return r;
    452        }
    453
    454        @Override
    455        public boolean hasFloatingPointElements() {
    456                Class<?> cls = getElementClass();
    457                return cls == Float.class || cls == Double.class;
    458        }
    459
    460        @Override
    461        public int getElementsPerItem() {
    462                return InterfaceUtils.getElementsPerItem(getClass());
    463        }
    464
    465        @Override
    466        public int getItemBytes() {
    467                return InterfaceUtils.getItemBytes(getElementsPerItem(), getClass());
    468        }
    469
    470        @Override
    471        public int getSize() {
    472                return size;
    473        }
    474
    475        @Override
    476        public int[] getShape() {
    477                // make a copy of the dimensions data, and put that out
    478                if (shape == null) {
    479                        logger.warn("Shape is null!!!");
    480                        return new int[] {};
    481                }
    482                return shape.clone();
    483        }
    484
    485        @Override
    486        public int getRank() {
    487                return shape == null ? 0 : shape.length;
    488        }
    489
    490        @Override
    491        public int getNbytes() {
    492                return getSize() * getItemBytes();
    493        }
    494
    495        /**
    496         * Check for -1 placeholder in shape and replace if necessary
    497         * @param shape to use
    498         * @param size expected size
    499         */
    500        private void checkShape(int[] shape, int size) {
    501                if (shape == null) {
    502                        if (size == 0) {
    503                                return;
    504                        }
    505                        logger.error("New shape must not be null for nonzero-sized dataset");
    506                        throw new IllegalArgumentException("New shape must not be null for nonzero-sized dataset");
    507                }
    508                int rank = shape.length;
    509                int found = -1;
    510                int nsize = 1;
    511                for (int i = 0; i < rank; i++) {
    512                        int d = shape[i];
    513                        if (d == -1) {
    514                                if (found == -1) {
    515                                        found = i;
    516                                } else {
    517                                        logger.error("Can only have one -1 placeholder in shape");
    518                                        throw new IllegalArgumentException("Can only have one -1 placeholder in shape");
    519                                }
    520                        } else {
    521                                nsize *= d;
    522                        }
    523                }
    524                if (found >= 0) {
    525                        shape[found] = size/nsize;
    526                } else if (nsize != size && !(rank == 0 && size == 0)) {
    527                        logger.error("New shape is not same size as old shape");
    528                        throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape));
    529                }
    530        }
    531
    532        @Override
    533        public void setShape(final int... shape) {
    534                int[] nshape = shape == null ? null : shape.clone();
    535                checkShape(nshape, size);
    536                if (Arrays.equals(this.shape, nshape)) {
    537                        return;
    538                }
    539
    540                if (stride != null) {
    541                        // the only compatible shapes are ones where new dimensions are factors of old dimensions
    542                        // or are combined adjacent old dimensions
    543                        int[] oshape = this.shape;
    544                        int orank = oshape.length;
    545                        int nrank = nshape.length;
    546                        int diff = nrank - orank;
    547                        int[] nstride = new int[nrank];
    548                        boolean ones = true;
    549                        // work forwards for broadcasting cases
    550                        for (int i = 0, j = 0; i < orank || j < nrank;) {
    551                                if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) {
    552                                        nstride[j++] = stride[i++];
    553                                } else if (j < nrank && nshape[j] == 1) {
    554                                        nstride[j++] = 0;
    555                                } else if (i < orank && oshape[i] == 1) {
    556                                        i++;
    557                                } else {
    558                                        if (j < nrank)
    559                                                j++;
    560                                        if (i < orank)
    561                                                i++;
    562                                        ones = false;
    563                                }
    564                        }
    565                        if (!ones) { // not just ones differ in shapes
    566                                int[] ostride = stride;
    567                                int ob = 0;
    568                                int oe = 1;
    569                                int nb = 0;
    570                                int ne = 1;
    571                                while (ob < orank && nb < nrank) {
    572                                        int ol = oshape[ob];
    573                                        int nl = nshape[nb];
    574
    575                                        if (nl < ol) { // find group of shape dimensions that form common size
    576                                                do { // case where new shape spreads single dimension over several dimensions
    577                                                        if (ne == nrank) {
    578                                                                break;
    579                                                        }
    580                                                        nl *= nshape[ne++];
    581                                                } while (nl < ol);
    582                                                if (nl != ol) {
    583                                                        logger.error("Subshape is incompatible with single dimension");
    584                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
    585                                                }
    586                                                int on = ne - 1;
    587                                                while (nshape[on] == 1) {
    588                                                        on--;
    589                                                }
    590
    591                                                nstride[on] = ostride[ob];
    592                                                for (int n = on - 1; n >= nb; n--) {
    593                                                        if (nshape[n] == 1)
    594                                                                continue;
    595
    596                                                        nstride[n] = nshape[on] * nstride[on];
    597                                                        on = n;
    598                                                }
    599                                        } else if (ol < nl) {
    600                                                do { // case where new shape combines several dimensions into one dimension
    601                                                        if (oe == orank) {
    602                                                                break;
    603                                                        }
    604                                                        ol *= oshape[oe++];
    605                                                } while (ol < nl);
    606                                                if (nl != ol) {
    607                                                        logger.error("Single dimension is incompatible with subshape");
    608                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
    609                                                }
    610
    611                                                int oo = oe - 1;
    612                                                while (oshape[oo] == 1) {
    613                                                        oo--;
    614                                                }
    615                                                int os = ostride[oo];
    616                                                for (int o = oo - 1; o >= ob; o--) {
    617                                                        if (oshape[o] == 1)
    618                                                                continue;
    619                                                        if (ostride[o] != oshape[oo] * ostride[oo]) {
    620                                                                logger.error("Subshape cannot be a non-contiguous view");
    621                                                                throw new IllegalArgumentException("Subshape cannot be a non-contiguous view");
    622                                                        }
    623                                                        oo = o;
    624                                                }
    625                                                nstride[nb] = os;
    626                                        } else {
    627                                                nstride[nb] = ostride[ob];
    628                                        }
    629
    630                                        ob = oe++;
    631                                        nb = ne++;
    632                                }
    633                        }
    634
    635                        stride = nstride;
    636                }
    637
    638                setDirty();
    639                if (this.shape != null && metadata != null) {
    640                        reshapeMetadata(this.shape, nshape);
    641                }
    642                this.shape = nshape;
    643        }
    644
    645        @Override
    646        public int[] getShapeRef() {
    647                return shape;
    648        }
    649
    650        @Override
    651        public int getOffset() {
    652                return offset;
    653        }
    654
    655        @Override
    656        public int[] getStrides() {
    657                return stride;
    658        }
    659
    660        @Override
    661        public Serializable getBuffer() {
    662                return odata;
    663        }
    664
    665        @Override
    666        public void overrideInternal(Serializable buffer, int... shape) {
    667                if (buffer != null) {
    668                        odata = buffer;
    669                        setData();
    670                        setDirty();
    671                }
    672
    673                if (shape != null) {
    674                        this.shape = shape.clone();
    675                        size = ShapeUtils.calcSize(this.shape);
    676                }
    677        }
    678
    679        /**
    680         * Create a stride array from dataset
    681         * @param a dataset
    682         * @param offset output offset
    683         * @return new strides
    684         */
    685        public static int[] createStrides(Dataset a, final int[] offset) {
    686                return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset);
    687        }
    688
    689        /**
    690         * Create a stride array from dataset
    691         * @param isize item size
    692         * @param shape to use
    693         * @param oStride original stride
    694         * @param oOffset original offset (only used if there is an original stride)
    695         * @param offset output offset
    696         * @return new strides
    697         */
    698        public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) {
    699                int rank = shape.length;
    700                final int[] stride;
    701                if (oStride == null) {
    702                        offset[0] = 0;
    703                        stride = new int[rank];
    704                        int s = isize;
    705                        for (int j = rank - 1; j >= 0; j--) {
    706                                stride[j] = s;
    707                                s *= shape[j];
    708                        }
    709                } else {
    710                        offset[0] = oOffset;
    711                        stride = oStride.clone();
    712                }
    713                return stride;
    714        }
    715
    716        /**
    717         * Create a stride array from slice information and a dataset
    718         * @param slice an n-D slice
    719         * @param a dataset
    720         * @param stride output stride
    721         * @param offset output offset
    722         * @return new shape
    723         */
    724        public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) {
    725                return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset);
    726        }
    727
    728        /**
    729         * Create a stride array from slice and dataset information
    730         * @param slice an n-D slice
    731         * @param isize item size
    732         * @param shape to use
    733         * @param oStride original stride
    734         * @param oOffset original offset (only used if there is an original stride)
    735         * @param stride output stride
    736         * @param offset output offset
    737         * @return new shape
    738         */
    739        public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) {
    740                int[] lstart = slice.getStart();
    741                int[] lstep = slice.getStep();
    742                int[] newShape = slice.getShape();
    743                int rank = shape.length;
    744
    745                if (oStride == null) {
    746                        int s = isize;
    747                        offset[0] = 0;
    748                        for (int j = rank - 1; j >= 0; j--) {
    749                                stride[j] = s * lstep[j];
    750                                offset[0] += s * lstart[j];
    751                                s *= shape[j];
    752                        }
    753                } else {
    754                        offset[0] = oOffset;
    755                        for (int j = 0; j < rank; j++) {
    756                                int s = oStride[j];
    757                                stride[j] = lstep[j] * s;
    758                                offset[0] += lstart[j] * s;
    759                        }
    760                }
    761
    762                return newShape;
    763        }
    764
    765        @Override
    766        public Dataset getBroadcastView(int... broadcastShape) {
    767                AbstractDataset view = getView(true);
    768
    769                if (!Arrays.equals(shape, broadcastShape)) {
    770                        List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape);
    771                        view.setShape(nShapes.get(0));
    772                        view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape);
    773                        view.base = this;
    774                        view.shape = broadcastShape.clone();
    775                        view.size = ShapeUtils.calcSize(broadcastShape);
    776                        if (view.name == null || view.name.isEmpty()) {
    777                                view.name = "Broadcast from " + Arrays.toString(shape);
    778                        } else {
    779                                view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape);
    780                        }
    781                }
    782                return view;
    783        }
    784
    785        @Override
    786        public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) {
    787                return internalGetSliceView(new SliceND(shape, start, stop, step));
    788        }
    789
    790        @Override
    791        public Dataset getSliceView(Slice... slice) {
    792                if (slice == null || slice.length == 0) {
    793                        return getView(true);
    794                }
    795
    796                return internalGetSliceView(new SliceND(shape, slice));
    797        }
    798
    799        /**
    800         * Get a slice of the dataset. The returned dataset is a view on a selection of items
    801         * @param slice an n-D slice
    802         * @return slice view
    803         */
    804        @Override
    805        public Dataset getSliceView(SliceND slice) {
    806                if (slice != null) {
    807                        checkSliceND(slice);
    808                }
    809                return internalGetSliceView(slice);
    810        }
    811
    812        private Dataset internalGetSliceView(SliceND slice) {
    813                if (slice.isAll()) {
    814                        return getView(true);
    815                }
    816
    817                final int rank = shape.length;
    818                int[] sStride = new int[rank];
    819                int[] sOffset = new int[1];
    820
    821                int[] sShape = createStrides(slice, this, sStride, sOffset);
    822
    823                AbstractDataset s = getView(false);
    824                s.shape = sShape;
    825                s.size = ShapeUtils.calcSize(sShape);
    826                s.stride = sStride;
    827                s.offset = sOffset[0];
    828                s.base = this;
    829
    830                s.metadata = copyMetadata();
    831                s.sliceMetadata(true, slice);
    832
    833                s.setDirty();
    834                s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE);
    835
    836                return s;
    837        }
    838
    839        /**
    840         * Get flattened view index of given position
    841         * @param pos
    842         *            the integer array specifying the n-D position
    843         * @return the index on the flattened dataset
    844         */
    845        private int getFlat1DIndex(final int[] pos) {
    846                final int imax = pos.length;
    847                if (imax == 0) {
    848                        return 0;
    849                }
    850
    851                return get1DIndexFromShape(pos);
    852        }
    853
    854        /**
    855         * @return index of first element
    856         * @since 2.0
    857         */
    858        protected int getFirst1DIndex() {
    859                if (shape == null) {
    860                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    861                }
    862                return stride == null ? 0 : offset;
    863        }
    864
    865        @Override
    866        public int get1DIndex(final int... n) {
    867                if (n.length == 0 && shape.length == 0)
    868                        return offset;
    869
    870                return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n);
    871        }
    872
    873        private static void throwAIOOBException(int i, int s, int d) {
    874                throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s
    875                                + "] in dimension " + d);
    876        }
    877
    878        /**
    879         * @param i position in first dimension
    880         * @return the index on the data array corresponding to that location
    881         */
    882        protected int get1DIndex(int i) {
    883                if (shape == null) {
    884                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    885                }
    886                if (shape.length > 1) {
    887                        logger.error("This dataset is not 1D but was addressed as such");
    888                        throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such");
    889                }
    890                if (i < 0) {
    891                        i += shape[0];
    892                }
    893                if (i < 0 || i >= shape[0]) {
    894                        throwAIOOBException(i, shape[0], 0);
    895                }
    896                return stride == null ? i : i*stride[0] + offset;
    897        }
    898
    899        /**
    900         * @param i position in first dimension
    901         * @param j position in second dimension
    902         * @return the index on the data array corresponding to that location
    903         */
    904        protected int get1DIndex(int i, int j) {
    905                if (shape == null) {
    906                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    907                }
    908                if (shape.length != 2) {
    909                        logger.error("This dataset is not 2D but was addressed as such");
    910                        throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such");
    911                }
    912                if (i < 0) {
    913                        i += shape[0];
    914                }
    915                if (i < 0 || i >= shape[0]) {
    916                        throwAIOOBException(i, shape[0], 0);
    917                }
    918                if (j < 0) {
    919                        j += shape[1];
    920                }
    921                if (j < 0 || j >= shape[1]) {
    922                        throwAIOOBException(i, shape[1], 1);
    923                }
    924                return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset;
    925        }
    926
    927        protected int get1DIndexFromShape(final int[] n) {
    928                return get1DIndexFromShape(shape, n);
    929        }
    930
    931        protected static int get1DIndexFromShape(final int[] shape, final int[] n) {
    932                if (shape == null) {
    933                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    934                }
    935                final int rank = shape.length;
    936                if (rank != n.length) {
    937                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
    938                        logger.error(errMsg);
    939                        throw new IllegalArgumentException(errMsg);
    940                }
    941                int index = 0;
    942                for (int i = 0; i < rank; i++) {
    943                        final int si = shape[i];
    944                        int ni = n[i];
    945                        if (ni < 0) {
    946                                ni += si;
    947                        }
    948                        if (ni < 0 || ni >= si) {
    949                                throwAIOOBException(ni, si, i);
    950                        }
    951                        index = index * si + ni;
    952                }
    953
    954                return index;
    955        }
    956
    957        private int get1DIndexFromStrides(final int[] n) {
    958                return get1DIndexFromStrides(shape, stride, offset, n);
    959        }
    960
    961        private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) {
    962                if (shape == null) {
    963                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    964                }
    965                final int rank = shape.length;
    966                if (rank != n.length) {
    967                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
    968                        logger.error(errMsg);
    969                        throw new IllegalArgumentException(errMsg);
    970                }
    971                int index = offset;
    972                for (int i = 0; i < rank; i++) {
    973                        final int st = stride[i];
    974                        if (st != 0) { // not broadcasted
    975                                final int si = shape[i];
    976                                int ni = n[i];
    977                                if (ni < 0) {
    978                                        ni += si;
    979                                }
    980                                if (ni < 0 || ni >= si) {
    981                                        throwAIOOBException(ni, si, i);
    982                                }
    983                                index += st * ni;
    984                        }
    985                }
    986                return index;
    987        }
    988
    989        @Override
    990        public int[] getNDPosition(final int n) {
    991                if (isIndexInRange(n)) {
    992                        throw new IllegalArgumentException("Index provided " + n
    993                                        + "is larger then the size of the containing array");
    994                }
    995
    996                return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n);
    997        }
    998
    999        private boolean isIndexInRange(final int n) {
    1000                if (stride == null) {
    1001                        return n >= size;
    1002                }
    1003                return n >= getBufferLength();
    1004        }
    1005
    1006        /**
    1007         * @return entire buffer length
    1008         */
    1009        abstract protected int getBufferLength();
    1010
    1011        private int[] getNDPositionFromStrides(int n) {
    1012                n -= offset;
    1013                int rank = shape.length;
    1014                if (rank == 1) {
    1015                        return new int[] { n / stride[0] };
    1016                }
    1017
    1018                int[] output = new int[rank];
    1019                int i = 0;
    1020                while (i != n) { // TODO find more efficient way than this exhaustive search
    1021                        int j = rank - 1;
    1022                        for (; j >= 0; j--) {
    1023                                output[j]++;
    1024                                i += stride[j];
    1025                                if (output[j] >= shape[j]) {
    1026                                        output[j] = 0;
    1027                                        i -= shape[j] * stride[j];
    1028                                } else {
    1029                                        break;
    1030                                }
    1031                        }
    1032                        if (j == -1) {
    1033                                logger.error("Index was not found in this strided dataset");
    1034                                throw new IllegalArgumentException("Index was not found in this strided dataset");
    1035                        }
    1036                }
    1037
    1038                return output;
    1039        }
    1040
    1041        @Override
    1042        public int checkAxis(int axis) {
    1043                return ShapeUtils.checkAxis(shape.length, axis);
    1044        }
    1045
    1046        @Deprecated
    1047        protected static int checkAxis(int rank, int axis) {
    1048                return ShapeUtils.checkAxis(rank, axis);
    1049        }
    1050
    1051        protected static final char BLOCK_OPEN = '[';
    1052        protected static final char BLOCK_CLOSE = ']';
    1053        protected final static String ELLIPSIS = "...";
    1054
    1055        @Override
    1056        public String toString() {
    1057                final int rank = shape == null ? 0 : shape.length;
    1058                final StringBuilder out = new StringBuilder();
    1059
    1060                if (InterfaceUtils.isElemental(getClass())) {
    1061                        out.append("Dataset ");
    1062                } else {
    1063                        out.append("Compound dataset (");
    1064                        out.append(getElementsPerItem());
    1065                        out.append(") ");
    1066                }
    1067
    1068                if (name != null && name.length() > 0) {
    1069                        out.append("'");
    1070                        out.append(name);
    1071                        out.append("' has shape ");
    1072                } else {
    1073                        out.append("shape is ");
    1074                }
    1075
    1076                out.append(BLOCK_OPEN);
    1077                if (rank > 0) {
    1078                        out.append(shape[0]);
    1079                }
    1080                for (int i = 1; i < rank; i++) {
    1081                        out.append(", " + shape[i]);
    1082                }
    1083                out.append(BLOCK_CLOSE);
    1084                return out.toString();
    1085        }
    1086
    1087        @Override
    1088        public String toString(boolean showData) {
    1089                if (!showData) {
    1090                        return toString();
    1091                }
    1092
    1093                if (size == 0) {
    1094                        return "[]";
    1095                }
    1096
    1097                final int rank = shape == null ? 0 : shape.length;
    1098                final StringBuilder out = new StringBuilder();
    1099
    1100                if (rank > 0) {
    1101                        int[] pos = new int[rank];
    1102                        final StringBuilder lead = new StringBuilder();
    1103                        printBlocks(out, lead, 0, pos);
    1104                } else {
    1105                        out.append(getString());
    1106                }
    1107                return out.toString();
    1108        }
    1109
    1110        /**
    1111         * Limit to strings output via the toString() method
    1112         */
    1113        private static int maxStringLength = 120;
    1114
    1115        /**
    1116         * Set maximum line length for toString() method
    1117         * @param maxLineLength limit on length of line
    1118         */
    1119        public static void setMaxLineLength(int maxLineLength) {
    1120                maxStringLength = maxLineLength;
    1121        }
    1122
    1123        /**
    1124         * @return maximum line length for toString() method
    1125         */
    1126        public static int getMaxLineLength() {
    1127                return maxStringLength;
    1128        }
    1129
    1130        /**
    1131         * Limit to number of sub-blocks output via the toString() method
    1132         */
    1133        private static final int MAX_SUBBLOCKS = 6;
    1134
    1135        private final static String SEPARATOR = ",";
    1136        private final static String SPACE = " ";
    1137        private final static String NEWLINE = "\n";
    1138
    1139        /**
    1140         * Make a line of output for last dimension of dataset
    1141         * @param end
    1142         * @param start
    1143         * @return line
    1144         */
    1145        private StringBuilder makeLine(final int end, final int[] start) {
    1146                StringBuilder line = new StringBuilder();
    1147                final int[] pos;
    1148                if (end >= start.length) {
    1149                        pos = Arrays.copyOf(start, end + 1);
    1150                } else {
    1151                        pos = start;
    1152                }
    1153                pos[end] = 0;
    1154                line.append(BLOCK_OPEN);
    1155                line.append(getString(pos));
    1156
    1157                final int length = shape[end];
    1158
    1159                // trim elements printed if length exceed estimate of maximum elements
    1160                int excess = length - maxStringLength / 3; // space + number + separator
    1161                int midIndex = -1;
    1162                if (excess > 0) {
    1163                        int index = (length - excess) / 2;
    1164                        for (int y = 1; y < index; y++) {
    1165                                line.append(SEPARATOR + SPACE);
    1166                                pos[end] = y;
    1167                                line.append(getString(pos));
    1168                        }
    1169                        midIndex = line.length() + 2;
    1170                        index = (length + excess) / 2;
    1171                        for (int y = index; y < length; y++) {
    1172                                line.append(SEPARATOR + SPACE);
    1173                                pos[end] = y;
    1174                                line.append(getString(pos));
    1175                        }
    1176                } else {
    1177                        for (int y = 1; y < length; y++) {
    1178                                line.append(SEPARATOR + SPACE);
    1179                                pos[end] = y;
    1180                                line.append(getString(pos));
    1181                        }
    1182                }
    1183                line.append(BLOCK_CLOSE);
    1184
    1185                // trim string down to limit
    1186                int lineLength = line.length();
    1187                excess = lineLength - maxStringLength - ELLIPSIS.length() - 1;
    1188                if (excess > 0) {
    1189                        int index = (lineLength - excess) / 2;
    1190                        if (midIndex > 0 && index > midIndex) {
    1191                                index = midIndex;
    1192                        } else {
    1193                                index = line.lastIndexOf(SEPARATOR, index) + 2;
    1194                        }
    1195                        StringBuilder out = new StringBuilder(line.subSequence(0, index));
    1196                        out.append(ELLIPSIS + SEPARATOR);
    1197                        index = (lineLength + excess) / 2;
    1198                        if (midIndex > 0 && index <= midIndex) {
    1199                                index = midIndex - 1;
    1200                        } else {
    1201                                index = line.indexOf(SEPARATOR, index) + 1;
    1202                        }
    1203                        out.append(line.subSequence(index, lineLength));
    1204                        return out;
    1205                } else if (midIndex > 0) { // add ellipsis
    1206                        StringBuilder out = new StringBuilder(line.subSequence(0, midIndex));
    1207                        out.append(ELLIPSIS + SEPARATOR + SPACE);
    1208                        out.append(line.subSequence(midIndex, lineLength));
    1209                        return out;
    1210                }
    1211
    1212                return line;
    1213        }
    1214
    1215        /**
    1216         * recursive method to print blocks
    1217         */
    1218        private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) {
    1219                if (out.length() > 0) {
    1220                        char last = out.charAt(out.length() - 1);
    1221                        if (last != BLOCK_OPEN) {
    1222                                out.append(lead);
    1223                        }
    1224                }
    1225                final int end = getRank() - 1;
    1226                if (level != end) {
    1227                        out.append(BLOCK_OPEN);
    1228                        int length = shape[level];
    1229
    1230                        // first sub-block
    1231                        pos[level] = 0;
    1232                        StringBuilder newlead = new StringBuilder(lead);
    1233                        newlead.append(SPACE);
    1234                        printBlocks(out, newlead, level + 1, pos);
    1235                        if (length < 2) { // escape
    1236                                out.append(BLOCK_CLOSE);
    1237                                return;
    1238                        }
    1239
    1240                        out.append(SEPARATOR + NEWLINE);
    1241                        for (int i = level + 1; i < end; i++) {
    1242                                out.append(NEWLINE);
    1243                        }
    1244
    1245                        // middle sub-blocks
    1246                        if (length < MAX_SUBBLOCKS) {
    1247                                for (int x = 1; x < length - 1; x++) {
    1248                                        pos[level] = x;
    1249                                        printBlocks(out, newlead, level + 1, pos);
    1250                                        if (end <= level + 1) {
    1251                                                out.append(SEPARATOR + NEWLINE);
    1252                                        } else {
    1253                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
    1254                                        }
    1255                                }
    1256                        } else {
    1257                                final int excess = length - MAX_SUBBLOCKS;
    1258                                int xmax = (length - excess) / 2;
    1259                                for (int x = 1; x < xmax; x++) {
    1260                                        pos[level] = x;
    1261                                        printBlocks(out, newlead, level + 1, pos);
    1262                                        if (end <= level + 1) {
    1263                                                out.append(SEPARATOR + NEWLINE);
    1264                                        } else {
    1265                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
    1266                                        }
    1267                                }
    1268                                out.append(newlead);
    1269                                out.append(ELLIPSIS + SEPARATOR + NEWLINE);
    1270                                xmax = (length + excess) / 2;
    1271                                for (int x = xmax; x < length - 1; x++) {
    1272                                        pos[level] = x;
    1273                                        printBlocks(out, newlead, level + 1, pos);
    1274                                        if (end <= level + 1) {
    1275                                                out.append(SEPARATOR + NEWLINE);
    1276                                        } else {
    1277                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
    1278                                        }
    1279                                }
    1280                        }
    1281
    1282                        // last sub-block
    1283                        pos[level] = length - 1;
    1284                        printBlocks(out, newlead, level + 1, pos);
    1285                        out.append(BLOCK_CLOSE);
    1286                } else {
    1287                        out.append(makeLine(end, pos));
    1288                }
    1289        }
    1290
    1291        @Override
    1292        public Dataset squeezeEnds() {
    1293                return squeeze(true);
    1294        }
    1295
    1296        @Override
    1297        public Dataset squeeze() {
    1298                return squeeze(false);
    1299        }
    1300
    1301        @Override
    1302        public Dataset squeeze(boolean onlyFromEnds) {
    1303                final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds);
    1304                final int[] oshape = shape;
    1305                if (stride == null) {
    1306                        shape = tshape;
    1307                } else {
    1308                        int rank = shape.length;
    1309                        int trank = tshape.length;
    1310                        if (trank < rank) {
    1311                                int[] tstride = new int[tshape.length];
    1312                                if (onlyFromEnds) {
    1313                                        for (int i = 0; i < rank; i++) {
    1314                                                if (shape[i] != 1) {
    1315                                                        for (int k = 0; k < trank; k++) {
    1316                                                                tstride[k] = stride[i++];
    1317                                                        }
    1318                                                        break;
    1319                                                }
    1320                                        }
    1321                                } else {
    1322                                        int t = 0;
    1323                                        for (int i = 0; i < rank; i++) {
    1324                                                if (shape[i] != 1) {
    1325                                                        tstride[t++] = stride[i];
    1326                                                }
    1327                                        }
    1328                                }
    1329                                shape = tshape;
    1330                                stride = tstride;
    1331                        }
    1332                }
    1333
    1334                setDirty();
    1335                reshapeMetadata(oshape, shape);
    1336                return this;
    1337        }
    1338
    1339        @Override
    1340        public boolean isCompatibleWith(final ILazyDataset g) {
    1341                return ShapeUtils.areShapesCompatible(shape, g.getShape());
    1342        }
    1343
    1344        @Override
    1345        public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException {
    1346                ShapeUtils.checkCompatibility(this, g);
    1347        }
    1348
    1349        @Override
    1350        public Dataset reshape(final int... shape) {
    1351                Dataset a;
    1352                try {
    1353                        a = getView(true);
    1354                        a.setShape(shape);
    1355                } catch (IllegalArgumentException e) {
    1356                        a = clone();
    1357                        a.setShape(shape);
    1358                }
    1359                return a;
    1360        }
    1361
    1362        /**
    1363         * @param start begin
    1364         * @param stop exclusive end
    1365         * @param step number to skip
    1366         * @return number of steps to take
    1367         */
    1368        protected static int calcSteps(final double start, final double stop, final double step) {
    1369                return Math.max(0, (int) Math.ceil((stop - start) / step));
    1370        }
    1371
    1372        @Override
    1373        public boolean isComplex() {
    1374                return false;
    1375        }
    1376
    1377        @Override
    1378        public Dataset getRealPart() {
    1379                return this;
    1380        }
    1381
    1382        @Override
    1383        public Dataset getRealView() {
    1384                return getView(true);
    1385        }
    1386
    1387        @Override
    1388        public Dataset getSlice(final int[] start, final int[] stop, final int[] step) {
    1389                return internalGetSlice(new SliceND(shape, start, stop, step));
    1390        }
    1391
    1392        @Override
    1393        public Dataset getSlice(Slice... slice) {
    1394                return internalGetSlice(new SliceND(shape, slice));
    1395        }
    1396
    1397        @Override
    1398        public Dataset getSlice(IMonitor monitor, Slice... slice) {
    1399                return getSlice(slice);
    1400        }
    1401
    1402        @Override
    1403        public Dataset getSlice(IMonitor monitor, SliceND slice) {
    1404                return getSlice(slice);
    1405        }
    1406
    1407        @Override
    1408        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) {
    1409                return getSlice(start, stop, step);
    1410        }
    1411
    1412        /**
    1413         * Get a slice of the dataset. The returned dataset is a copied selection of items
    1414         * @param slice an n-D slice
    1415         * @return The dataset of the sliced data
    1416         */
    1417        @Override
    1418        public Dataset getSlice(final SliceND slice) {
    1419                if (slice != null) {
    1420                        checkSliceND(slice);
    1421                }
    1422                return internalGetSlice(slice);
    1423        }
    1424
    1425        private Dataset internalGetSlice(final SliceND slice) {
    1426                SliceIterator it = (SliceIterator) internalGetSliceIterator(slice);
    1427                AbstractDataset s = getSlice(it);
    1428                s.metadata = copyMetadata();
    1429                s.setDirty();
    1430                s.sliceMetadata(true, slice);
    1431                return s;
    1432        }
    1433
    1434        /**
    1435         * Get a slice of the dataset. The returned dataset is a copied selection of items
    1436         *
    1437         * @param iterator Slice iterator
    1438         * @return The dataset of the sliced data
    1439         */
    1440        abstract public AbstractDataset getSlice(final SliceIterator iterator);
    1441
    1442        @Override
    1443        public Dataset setSlice(final Object obj, final SliceND slice) {
    1444                if (slice != null) {
    1445                        checkSliceND(slice);
    1446                }
    1447                return internalSetSlice(obj, slice);
    1448        }
    1449
    1450        private Dataset internalSetSlice(final Object obj, final SliceND slice) {
    1451                Dataset ds;
    1452                if (obj instanceof Dataset) {
    1453                        ds = (Dataset) obj;
    1454                } else if (obj instanceof IDataset) {
    1455                        ds = DatasetUtils.convertToDataset((IDataset) obj);
    1456                } else {
    1457                        Class<? extends Dataset> dClass = getClass();
    1458                        if (!BooleanDataset.class.equals(dClass)) {
    1459                                dClass = InterfaceUtils.getLargestInterface(this);
    1460                        }
    1461                        ds = DatasetFactory.createFromObject(getElementsPerItem(), dClass, obj);
    1462                }
    1463
    1464                return setSlicedView(getSliceView(slice), ds);
    1465        }
    1466
    1467        @Override
    1468        public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) {
    1469                return internalSetSlice(obj, new SliceND(shape, start, stop, step));
    1470        }
    1471
    1472        /**
    1473         * Set a view of current dataset to given dataset with broadcasting
    1474         * @param view destination
    1475         * @param d source of data
    1476         * @return this dataset
    1477         */
    1478        abstract Dataset setSlicedView(Dataset view, Dataset d);
    1479
    1480        @Override
    1481        public Dataset setSlice(Object obj, Slice... slice) {
    1482                if (slice == null || slice.length == 0) {
    1483                        return internalSetSlice(obj, new SliceND(shape));
    1484                }
    1485                return internalSetSlice(obj, new SliceND(shape, slice));
    1486        }
    1487
    1488        @Override
    1489        public boolean all() {
    1490                return Comparisons.allTrue(this);
    1491        }
    1492
    1493        @Override
    1494        public BooleanDataset all(final int axis) {
    1495                return Comparisons.allTrue(this, axis);
    1496        }
    1497
    1498        @Override
    1499        public boolean any() {
    1500                return Comparisons.anyTrue(this);
    1501        }
    1502
    1503        @Override
    1504        public BooleanDataset any(final int axis) {
    1505                return Comparisons.anyTrue(this, axis);
    1506        }
    1507
    1508        @Override
    1509        public Dataset ifloorDivide(final Object o) {
    1510                return idivide(o).ifloor();
    1511        }
    1512
    1513        @Override
    1514        public double residual(final Object o) {
    1515                return residual(o, null, false);
    1516        }
    1517
    1518        @Override
    1519        public double residual(final Object o, boolean ignoreNaNs) {
    1520                return residual(o, null, ignoreNaNs);
    1521        }
    1522
    1523        /**
    1524         * @return statistics
    1525         * @since 2.0
    1526         */
    1527        @SuppressWarnings("unchecked")
    1528        protected StatisticsMetadata<Number> getStats() {
    1529                StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class);
    1530                if (md == null || md.isDirty(this)) {
    1531                        md = new StatisticsMetadataImpl<Number>();
    1532                        md.initialize(this);
    1533                        setMetadata(md);
    1534                }
    1535                return md;
    1536        }
    1537
    1538        /**
    1539         * @return statistics
    1540         * @since 2.0
    1541         */
    1542        @SuppressWarnings("unchecked")
    1543        protected StatisticsMetadata<String> getStringStats() {
    1544                StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class);
    1545                if (md == null || md.isDirty(this)) {
    1546                        md = new StatisticsMetadataImpl<String>();
    1547                        md.initialize(this);
    1548                        setMetadata(md);
    1549                }
    1550                return md;
    1551        }
    1552
    1553        @Override
    1554        public Number max(boolean... ignoreInvalids) {
    1555                return getStats().getMaximum(ignoreInvalids);
    1556        }
    1557
    1558        @Override
    1559        public Dataset max(int axis, boolean... ignoreInvalids) {
    1560                return getStats().getMaximum(axis, ignoreInvalids);
    1561        }
    1562
    1563        @Override
    1564        public Dataset max(int[] axes, boolean... ignoreInvalids) {
    1565                return getStats().getMaximum(axes, ignoreInvalids);
    1566        }
    1567
    1568        @Override
    1569        public Number min(boolean... ignoreInvalids) {
    1570                return getStats().getMinimum(ignoreInvalids);
    1571        }
    1572
    1573        @Override
    1574        public Dataset min(int axis, boolean... ignoreInvalids) {
    1575                return getStats().getMinimum(axis, ignoreInvalids);
    1576        }
    1577
    1578        @Override
    1579        public Dataset min(int[] axes, boolean... ignoreInvalids) {
    1580                return getStats().getMinimum(axes, ignoreInvalids);
    1581        }
    1582
    1583        @Override
    1584        public int argMax(boolean... ignoreInvalids) {
    1585                return getFlat1DIndex(maxPos(ignoreInvalids));
    1586        }
    1587
    1588        /**
    1589         * @since 2.0
    1590         */
    1591        @Override
    1592        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
    1593                return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids);
    1594        }
    1595
    1596        @Override
    1597        public int argMin(boolean... ignoreInvalids) {
    1598                return getFlat1DIndex(minPos(ignoreInvalids));
    1599        }
    1600
    1601        /**
    1602         * @since 2.0
    1603         */
    1604        @Override
    1605        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
    1606                return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids);
    1607        }
    1608
    1609        @Override
    1610        public Number peakToPeak(boolean... ignoreInvalids) {
    1611                return InterfaceUtils.fromDoubleToBiggestNumber(getClass(), max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue());
    1612        }
    1613
    1614        @Override
    1615        public Dataset peakToPeak(int axis,  boolean... ignoreInvalids) {
    1616                return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids));
    1617        }
    1618
    1619        @Override
    1620        public Dataset peakToPeak(int[] axes,  boolean... ignoreInvalids) {
    1621                return Maths.subtract(max(axes, ignoreInvalids), min(axes, ignoreInvalids));
    1622        }
    1623
    1624        @Override
    1625        public long count(boolean... ignoreInvalids) {
    1626                return getStats().getCount(ignoreInvalids);
    1627        }
    1628
    1629        @Override
    1630        public Dataset count(int axis, boolean... ignoreInvalids) {
    1631                return getStats().getCount(axis, ignoreInvalids);
    1632        }
    1633
    1634        @Override
    1635        public Dataset count(int[] axes, boolean... ignoreInvalids) {
    1636                return getStats().getCount(axes, ignoreInvalids);
    1637        }
    1638
    1639        @Override
    1640        public Object sum(boolean... ignoreInvalids) {
    1641                return InterfaceUtils.toBiggestNumber(getClass(), getStats().getSum(ignoreInvalids));
    1642        }
    1643
    1644        @Override
    1645        public Dataset sum(int axis, boolean... ignoreInvalids) {
    1646                return getStats().getSum(axis, ignoreInvalids);
    1647        }
    1648
    1649        @Override
    1650        public Dataset sum(int[] axes, boolean... ignoreInvalids) {
    1651                return getStats().getSum(axes, ignoreInvalids);
    1652        }
    1653
    1654        @Override
    1655        public Object product(boolean... ignoreInvalids) {
    1656                return Stats.product(this, ignoreInvalids);
    1657        }
    1658
    1659        @Override
    1660        public Dataset product(int axis, boolean... ignoreInvalids) {
    1661                return Stats.product(this, axis, ignoreInvalids);
    1662        }
    1663
    1664        @Override
    1665        public Dataset product(int[] axes, boolean... ignoreInvalids) {
    1666                return Stats.product(this, axes, ignoreInvalids);
    1667        }
    1668
    1669        @Override
    1670        public Object mean(boolean... ignoreInvalids) {
    1671                return getStats().getMean(ignoreInvalids);
    1672        }
    1673
    1674        @Override
    1675        public Dataset mean(int axis, boolean... ignoreInvalids) {
    1676                return getStats().getMean(axis, ignoreInvalids);
    1677        }
    1678
    1679        @Override
    1680        public Dataset mean(int[] axes, boolean... ignoreInvalids) {
    1681                return getStats().getMean(axes, ignoreInvalids);
    1682        }
    1683
    1684        @Override
    1685        public double variance() {
    1686                return variance(false);
    1687        }
    1688
    1689        @Override
    1690        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
    1691                return getStats().getVariance(isWholePopulation, ignoreInvalids);
    1692        }
    1693
    1694        @Override
    1695        public Dataset variance(int axis) {
    1696                return getStats().getVariance(axis, false);
    1697        }
    1698
    1699        @Override
    1700        public Dataset variance(int[] axes) {
    1701                return getStats().getVariance(axes, false);
    1702        }
    1703
    1704        @Override
    1705        public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    1706                return getStats().getVariance(axis, isWholePopulation, ignoreInvalids);
    1707        }
    1708
    1709        @Override
    1710        public Dataset variance(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) {
    1711                return getStats().getVariance(axes, isWholePopulation, ignoreInvalids);
    1712        }
    1713
    1714        @Override
    1715        public double stdDeviation() {
    1716                return Math.sqrt(variance());
    1717        }
    1718
    1719        @Override
    1720        public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) {
    1721                return Math.sqrt(variance(isWholePopulation, ignoreInvalids));
    1722        }
    1723
    1724        @Override
    1725        public Dataset stdDeviation(int axis) {
    1726                return Maths.sqrt(variance(axis, false));
    1727        }
    1728
    1729        @Override
    1730        public Dataset stdDeviation(int[] axes) {
    1731                return Maths.sqrt(variance(axes, false));
    1732        }
    1733
    1734        @Override
    1735        public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    1736                return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids));
    1737        }
    1738
    1739        @Override
    1740        public Dataset stdDeviation(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) {
    1741                return Maths.sqrt(variance(axes, isWholePopulation, ignoreInvalids));
    1742        }
    1743
    1744        @Override
    1745        public double rootMeanSquare(boolean... ignoreInvalids) {
    1746                StatisticsMetadata<Number> stats = getStats();
    1747                final double mean = stats.getMean(ignoreInvalids).doubleValue();
    1748                final double var = stats.getVariance(true, ignoreInvalids);
    1749                return Math.sqrt(var + mean * mean);
    1750        }
    1751
    1752        @Override
    1753        public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
    1754                StatisticsMetadata<Number> stats = getStats();
    1755                Dataset v = stats.getVariance(axis, true, ignoreInvalids);
    1756                Dataset m = stats.getMean(axis, ignoreInvalids);
    1757                Dataset result = Maths.multiply(m, m);
    1758                return Maths.sqrt(result.iadd(v));
    1759        }
    1760
    1761        @Override
    1762        public Dataset rootMeanSquare(int[] axes, boolean... ignoreInvalids) {
    1763                StatisticsMetadata<Number> stats = getStats();
    1764                Dataset v = stats.getVariance(axes, true, ignoreInvalids);
    1765                Dataset m = stats.getMean(axes, ignoreInvalids);
    1766                Dataset result = Maths.multiply(m, m);
    1767                return Maths.sqrt(result.iadd(v));
    1768        }
    1769
    1770        /**
    1771         * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards.
    1772         *
    1773         * @param dindex destination index
    1774         * @param sindex source index
    1775         * @param src
    1776         *            is the source data buffer
    1777         */
    1778        protected abstract void setItemDirect(final int dindex, final int sindex, final Object src);
    1779
    1780        /**
    1781         * @return error broadcasted to current shape
    1782         */
    1783        private Dataset getBroadcastedInternalError() {
    1784                if (shape == null) {
    1785                        throw new IllegalArgumentException("Cannot get error for null dataset");
    1786                }
    1787                ILazyDataset led = super.getErrors();
    1788                if (led == null)
    1789                        return null;
    1790
    1791                Dataset ed = null;
    1792                try {
    1793                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
    1794                } catch (DatasetException e) {
    1795                        logger.error("Could not get data from lazy dataset", e);
    1796                }
    1797                if (led != ed) {
    1798                        setErrors(ed); // set back
    1799                }
    1800
    1801                return ed.getBroadcastView(shape);
    1802        }
    1803
    1804        @Override
    1805        public Dataset getErrors() {
    1806                Dataset ed = getBroadcastedInternalError();
    1807                if (ed == null)
    1808                        return null;
    1809
    1810                return ed;
    1811        }
    1812
    1813        @Override
    1814        public double getError() {
    1815                Dataset ed = getBroadcastedInternalError();
    1816                if (ed == null)
    1817                        return 0;
    1818
    1819                return ed.getDouble();
    1820        }
    1821
    1822        @Override
    1823        public double getError(final int i) {
    1824                Dataset ed = getBroadcastedInternalError();
    1825                if (ed == null)
    1826                        return 0;
    1827
    1828                return ed.getDouble(i);
    1829        }
    1830
    1831        @Override
    1832        public double getError(final int i, final int j) {
    1833                Dataset ed = getBroadcastedInternalError();
    1834                if (ed == null)
    1835                        return 0;
    1836
    1837                return ed.getDouble(i, j);
    1838        }
    1839
    1840        @Override
    1841        public double getError(int... pos) {
    1842                Dataset ed = getBroadcastedInternalError();
    1843                if (ed == null)
    1844                        return 0;
    1845
    1846                return ed.getDouble(pos);
    1847        }
    1848
    1849        @Override
    1850        public double[] getErrorArray(final int i) {
    1851                Dataset ed = getBroadcastedInternalError();
    1852                if (ed == null)
    1853                        return null;
    1854
    1855                return new double[] {getError(i)};
    1856        }
    1857
    1858        @Override
    1859        public double[] getErrorArray(final int i, final int j) {
    1860                Dataset ed = getBroadcastedInternalError();
    1861                if (ed == null)
    1862                        return null;
    1863
    1864                return new double[] {getError(i, j)};
    1865        }
    1866
    1867        @Override
    1868        public double[] getErrorArray(int... pos) {
    1869                Dataset ed = getBroadcastedInternalError();
    1870                if (ed == null)
    1871                        return null;
    1872
    1873                return new double[] {getError(pos)};
    1874        }
    1875
    1876        protected Dataset getInternalSquaredError() {
    1877                Dataset sed = getErrorBuffer().getBroadcastView(shape);
    1878                return sed;
    1879        }
    1880
    1881        @Override
    1882        public Dataset getErrorBuffer() {
    1883                ErrorMetadata emd = getErrorMetadata();
    1884                if (emd == null)
    1885                        return null;
    1886
    1887                if (!(emd instanceof ErrorMetadataImpl)) {
    1888                        ILazyDataset led = emd.getError();
    1889                        Dataset ed;
    1890                        try {
    1891                                ed = DatasetUtils.sliceAndConvertLazyDataset(led);
    1892                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1893                                setMetadata(emd);
    1894                                emd.setError(ed);
    1895                        } catch (MetadataException me) {
    1896                                logger.error("Could not create metadata", me);
    1897                        } catch (DatasetException e) {
    1898                                logger.error("Could not get data from lazy dataset", e);
    1899                        }
    1900                }
    1901
    1902                return ((ErrorMetadataImpl) emd).getSquaredError();
    1903        }
    1904
    1905        /**
    1906         * Set a copy of the buffer that backs the (squared) error data
    1907         * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset
    1908         */
    1909        @Override
    1910        public void setErrorBuffer(Serializable buffer) {
    1911                if (shape == null) {
    1912                        throw new IllegalArgumentException("Cannot set error buffer for null dataset");
    1913                }
    1914                if (buffer == null) {
    1915                        clearMetadata(ErrorMetadata.class);
    1916                        return;
    1917                }
    1918
    1919                IDataset d = (IDataset) createFromSerializable(buffer, false);
    1920                ErrorMetadata emd = getErrorMetadata();
    1921                if (!(emd instanceof ErrorMetadataImpl)) {
    1922                        try {
    1923                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1924                                setMetadata(emd);
    1925                        } catch (MetadataException me) {
    1926                                logger.error("Could not create metadata", me);
    1927                        }
    1928                }
    1929                ((ErrorMetadataImpl) emd).setSquaredError(d);
    1930        }
    1931}