    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.Serializable;
    016import java.lang.reflect.Array;
    017import java.util.ArrayList;
    018import java.util.Arrays;
    019import java.util.Comparator;
    020import java.util.List;
    021
    022import org.apache.commons.math3.util.MathArrays;
    023import org.eclipse.january.DatasetException;
    024import org.slf4j.Logger;
    025import org.slf4j.LoggerFactory;
    026
    027/**
    028 * Utilities for manipulating datasets
    029 */
    030public class DatasetUtils {
    031
    032        /**
    033         * Setup the logging facilities
    034         */
    035        private static final Logger utilsLogger = LoggerFactory.getLogger(DatasetUtils.class);
    036
    037        /**
    038         * Append copy of dataset with another dataset along n-th axis
    039         *
    040         * @param a first dataset
    041         * @param b second dataset
    042         * @param axis
    043         *            number of axis (negative number counts from last)
    044         * @return appended dataset
    045         */
    046        public static Dataset append(IDataset a, IDataset b, int axis) {
    047                final int[] ashape = a.getShape();
    048                final int rank = ashape.length;
    049                final int[] bshape = b.getShape();
    050                if (rank != bshape.length) {
    051                        throw new IllegalArgumentException("Incompatible number of dimensions");
    052                }
    053                axis = ShapeUtils.checkAxis(rank, axis);
    054
    055                for (int i = 0; i < rank; i++) {
    056                        if (i != axis && ashape[i] != bshape[i]) {
    057                                throw new IllegalArgumentException("Incompatible dimensions");
    058                        }
    059                }
    060                final int[] nshape = new int[rank];
    061                for (int i = 0; i < rank; i++) {
    062                        nshape[i] = ashape[i];
    063                }
    064                nshape[axis] += bshape[axis];
    065                final Class<? extends Dataset> ot = InterfaceUtils.getInterface(b);
    066                final Class<? extends Dataset> dt = InterfaceUtils.getInterface(a);
    067                Dataset ds = DatasetFactory.zeros(a.getElementsPerItem(), InterfaceUtils.getBestInterface(ot, dt), nshape);
    068                IndexIterator iter = ds.getIterator(true);
    069                int[] pos = iter.getPos();
    070                while (iter.hasNext()) {
    071                        int d = ashape[axis];
    072                        if (pos[axis] < d) {
    073                                ds.setObjectAbs(iter.index, a.getObject(pos));
    074                        } else {
    075                                pos[axis] -= d;
    076                                ds.setObjectAbs(iter.index, b.getObject(pos));
    077                                pos[axis] += d;
    078                        }
    079                }
    080
    081                return ds;
    082        }
    083
    084        /**
    085         * Changes specific items of dataset by replacing them with other array
    086         * @param <T> dataset class
    087         * @param a dataset
    088         * @param indices dataset interpreted as integers where to put items
    089         * @param values to set
    090         * @return changed dataset
    091         */
    092        public static <T extends Dataset> T put(final T a, final Dataset indices, Object values) {
    093                IndexIterator it = indices.getIterator();
    094                Dataset vd = DatasetFactory.createFromObject(values).flatten();
    095                int vlen = vd.getSize();
    096                int v = 0;
    097                while (it.hasNext()) {
    098                        if (v >= vlen) v -= vlen;
    099
    100                        a.setObjectAbs((int) indices.getElementLongAbs(it.index), vd.getObjectAbs(v++));
    101                }
    102                return a;
    103        }
    104
    105        /**
    106         * Changes specific items of dataset by replacing them with other array
    107         * @param <T> dataset class
    108         * @param a dataset
    109         * @param indices where to put items
    110         * @param values to set
    111         * @return changed dataset
    112         */
    113        public static <T extends Dataset> T put(final T a, final int[] indices, Object values) {
    114                int ilen = indices.length;
    115                Dataset vd = DatasetFactory.createFromObject(values).flatten();
    116                int vlen = vd.getSize();
    117                for (int i = 0, v= 0; i < ilen; i++) {
    118                        if (v >= vlen) v -= vlen;
    119
    120                        a.setObjectAbs(indices[i], vd.getObjectAbs(v++));
    121                }
    122                return a;
    123        }
    124
    125        /**
    126         * Take items from dataset along an axis
    127         * @param <T> dataset class
    128         * @param a dataset
    129         * @param indices dataset interpreted as integers where to take items
    130         * @param axis if null, then use flattened view
    131         * @return a sub-dataset
    132         */
    133        public static <T extends Dataset> T take(final T a, final Dataset indices, Integer axis) {
    134                IntegerDataset indexes = indices.flatten().cast(IntegerDataset.class);
    135                return take(a, indexes.getData(), axis);
    136        }
    137
    138        /**
    139         * Take items from dataset along an axis
    140         * @param <T> dataset class
    141         * @param a dataset
    142         * @param indices where to take items
    143         * @param axis if null, then use flattened view
    144         * @return a sub-dataset
    145         */
    146        public static <T extends Dataset> T take(final T a, final int[] indices, Integer axis) {
    147                if (indices == null || indices.length == 0) {
    148                        utilsLogger.error("No indices given");
    149                        throw new IllegalArgumentException("No indices given");
    150                }
    151                int[] ashape = a.getShape();
    152                final int rank = ashape.length;
    153                final int ilen = indices.length;
    154
    155                T result;
    156                if (axis == null) {
    157                        ashape = new int[1];
    158                        ashape[0] = ilen;
    159                        result = DatasetFactory.zeros(a, ashape);
    160                        Serializable src = a.getBuffer();
    161                        for (int i = 0; i < ilen; i++) {
    162                                ((AbstractDataset) result).setItemDirect(i, indices[i], src);
    163                        }
    164                } else {
    165                        axis = a.checkAxis(axis);
    166                        ashape[axis] = ilen;
    167                        result = DatasetFactory.zeros(a, ashape);
    168
    169                        int[] dpos = new int[rank];
    170                        int[] spos = new int[rank];
    171                        boolean[] axes = new boolean[rank];
    172                        Arrays.fill(axes, true);
    173                        axes[axis] = false;
    174                        Serializable src = a.getBuffer();
    175                        for (int i = 0; i < ilen; i++) {
    176                                spos[axis] = indices[i];
    177                                dpos[axis] = i;
    178                                SliceIterator siter = a.getSliceIteratorFromAxes(spos, axes);
    179                                SliceIterator diter = result.getSliceIteratorFromAxes(dpos, axes);
    180
    181                                while (siter.hasNext() && diter.hasNext()) {
    182                                        ((AbstractDataset) result).setItemDirect(diter.index, siter.index, src);
    183                                }
    184                        }
    185                }
    186                result.setDirty();
    187                return result;
    188        }
    189
    190        /**
    191         * Construct a dataset that contains the original dataset repeated the number
    192         * of times in each axis given by corresponding entries in the reps array
    193         *
    194         * @param a dataset
    195         * @param reps repetitions in each dimension
    196         * @return tiled dataset
    197         */
    198        public static Dataset tile(final IDataset a, int... reps) {
    199                int[] shape = a.getShape();
    200                int rank = shape.length;
    201                final int rlen = reps.length;
    202
    203                // expand shape
    204                if (rank < rlen) {
    205                        int[] newShape = new int[rlen];
    206                        int extraRank = rlen - rank;
    207                        for (int i = 0; i < extraRank; i++) {
    208                                newShape[i] = 1;
    209                        }
    210                        for (int i = 0; i < rank; i++) {
    211                                newShape[i+extraRank] = shape[i];
    212                        }
    213
    214                        shape = newShape;
    215                        rank = rlen;
    216                } else if (rank > rlen) {
    217                        int[] newReps = new int[rank];
    218                        int extraRank = rank - rlen;
    219                        for (int i = 0; i < extraRank; i++) {
    220                                newReps[i] = 1;
    221                        }
    222                        for (int i = 0; i < rlen; i++) {
    223                                newReps[i+extraRank] = reps[i];
    224                        }
    225                        reps = newReps;
    226                }
    227
    228                // calculate new shape
    229                int[] newShape = new int[rank];
    230                for (int i = 0; i < rank; i++) {
    231                        newShape[i] = shape[i]*reps[i];
    232                }
    233
    234                Dataset tdata = DatasetFactory.zeros(a.getElementsPerItem(), InterfaceUtils.getInterfaceFromClass(a.getElementsPerItem(), a.getElementClass()), newShape);
    235
    236                // decide which way to put slices
    237                boolean manyColumns;
    238                if (rank == 1)
    239                        manyColumns = true;
    240                else
    241                        manyColumns = shape[rank-1] > 64;
    242
    243                if (manyColumns) {
    244                        // generate each start point and put a slice in
    245                        IndexIterator iter = tdata.getSliceIterator(null, null, shape);
    246                        SliceIterator siter = (SliceIterator) tdata.getSliceIterator(null, shape, null);
    247                        final int[] pos = iter.getPos();
    248                        while (iter.hasNext()) {
    249                                siter.setStart(pos);
    250                                tdata.setSlice(a, siter);
    251                        }
    252
    253                } else {
    254                        // for each value, set slice given by repeats
    255                        final int[] skip = new int[rank];
    256                        for (int i = 0; i < rank; i++) {
    257                                if (reps[i] == 1) {
    258                                        skip[i] = newShape[i];
    259                                } else {
    260                                        skip[i] = shape[i];
    261                                }
    262                        }
    263
    264                        Dataset aa = convertToDataset(a);
    265                        IndexIterator ita = aa.getIterator(true);
    266                        final int[] pos = ita.getPos();
    267
    268                        final int[] sstart = new int[rank];
    269                        final int extra = rank - pos.length;
    270                        for (int i = 0; i < extra; i++) {
    271                                sstart[i] = 0;
    272                        }
    273                        SliceIterator siter = (SliceIterator) tdata.getSliceIterator(sstart, null, skip);
    274                        while (ita.hasNext()) {
    275                                for (int i = 0; i < pos.length; i++) {
    276                                        sstart[i + extra] = pos[i];
    277                                }
    278                                siter.setStart(sstart);
    279                                tdata.setSlice(aa.getObjectAbs(ita.index), siter);
    280                        }
    281                }
    282
    283                return tdata;
    284        }
    285
    286        /**
    287         * Permute copy of dataset's axes so that given order is old order:
    288         * <pre>{@literal
    289         *  axisPerm = (p(0), p(1),...) => newdata(n(0), n(1),...) = olddata(o(0), o(1), ...)
    290         *  such that n(i) = o(p(i)) for all i
    291         * }</pre>
    292         * I.e. for a 3D dataset (1,0,2) implies the new dataset has its 1st dimension
    293         * running along the old dataset's 2nd dimension and the new 2nd is the old 1st.
    294         * The 3rd dimension is left unchanged.
    295         *
    296         * @param a dataset
    297         * @param axes if null or zero length then axes order reversed
    298         * @return remapped copy of data
    299         */
    300        public static Dataset transpose(final IDataset a, int... axes) {
    301                return convertToDataset(a).transpose(axes);
    302        }
    303
    304        /**
    305         * Swap two axes in dataset
    306         * @param a dataset
    307         * @param axis1 first axis
    308         * @param axis2 second axis
    309         * @return swapped dataset
    310         */
    311        public static Dataset swapAxes(final IDataset a, int axis1, int axis2) {
    312                return convertToDataset(a).swapAxes(axis1, axis2);
    313        }
    314
    315        /**
    316         * @param <T> dataset class
    317         * @param a dataset
    318         * @return sorted flattened copy of dataset
    319         */
    320        public static <T extends Dataset> T sort(final T a) {
    321                return sort(a, (Integer) null);
    322        }
    323
    324        /**
    325         * @param <T> dataset class
    326         * @param a dataset
    327         * @param axis to sort along, if null then dataset is first flattened
    328         * @return dataset sorted along axis
    329         */
    330        @SuppressWarnings("unchecked")
    331        public static <T extends Dataset> T sort(final T a, final Integer axis) {
    332                T s = (T) a.clone();
    333                return (T) s.sort(axis);
    334        }
    335
    336        /**
    337         * Sort in place given dataset and reorder ancillary datasets too
    338         * @param a dataset to be sorted
    339         * @param b ancillary datasets
    340         */
    341        public static void sort(Dataset a, Dataset... b) {
    342                if (!InterfaceUtils.isNumerical(a.getClass())) {
    343                        throw new UnsupportedOperationException("Sorting non-numerical datasets not supported yet");
    344                }
    345
    346                // gather all datasets as double dataset copies
    347                DoubleDataset s = copy(DoubleDataset.class, a);
    348                int l = b == null ? 0 : b.length;
    349                DoubleDataset[] t = new DoubleDataset[l];
    350                int n = 0;
    351                for (int i = 0; i < l; i++) {
    352                        if (b[i] != null) {
    353                                if (!InterfaceUtils.isNumerical(b[i].getClass())) {
    354                                        throw new UnsupportedOperationException("Sorting non-numerical datasets not supported yet");
    355                                }
    356                                t[i] = copy(DoubleDataset.class, b[i]);
    357                                n++;
    358                        }
    359                }
    360
    361                double[][] y = new double[n][];
    362                for (int i = 0, j = 0; i < l; i++) {
    363                        if (t[i] != null) {
    364                                y[j++] = t[i].getData();
    365                        }
    366                }
    367
    368                MathArrays.sortInPlace(s.getData(), y);
    369
    370                a.setSlice(s);
    371                for (int i = 0; i < l; i++) {
    372                        if (b[i] != null) {
    373                                b[i].setSlice(t[i]);
    374                        }
    375                }
    376        }
    377
    378        /**
    379         * Indirectly sort along given axis
    380         * @param a dataset whose indexes will be sorted
    381         * @param axis to sort along, if null then dataset is first flattened
    382         * @return indexes
    383         * @since 2.1
    384         */
    385        public static IntegerDataset indexSort(Dataset a, Integer axis) {
    386                if (axis == null) {
    387                        int size = a.getSize();
    388                        Integer[] index = new Integer[size];
    389                        for (int i = 0; i < size; i++) {
    390                                index[i] = i;
    391                        }
    392                        final Dataset f = a.flatten(); // is this correct for views??? Check with NumPy
    393                        Comparator<Integer> cmp = new Comparator<Integer>() {
    394
    395                                @Override
    396                                public int compare(Integer o1, Integer o2) {
    397
    398                                        return Double.compare(f.getElementDoubleAbs(o1), f.getElementDoubleAbs(o2));
    399                                }
    400                        };
    401                        Arrays.sort(index, cmp);
    402                        return DatasetFactory.createFromObject(IntegerDataset.class, index);
    403                }
    404
    405                axis = a.checkAxis(axis);
    406                final int[] shape = a.getShapeRef();
    407                IntegerDataset id = DatasetFactory.zeros(IntegerDataset.class, shape);
    408                int size = shape[axis];
    409                Integer[] index = new Integer[size];
    410
    411                int[] dShape = new int[shape.length];
    412                Arrays.fill(dShape, 1);
    413                dShape[axis] = size;
    414                final DoubleDataset dd = DatasetFactory.zeros(DoubleDataset.class, dShape);
    415                final Comparator<Integer> cmp = new Comparator<Integer>() {
    416                        @Override
    417                        public int compare(Integer o1, Integer o2) {
    418
    419                                return Double.compare(dd.getElementDoubleAbs(o1), dd.getElementDoubleAbs(o2));
    420                        }
    421                };
    422
    423                SliceND ds = new SliceND(dShape);
    424                SliceNDIterator it = new SliceNDIterator(new SliceND(shape), axis);
    425                int[] pos = it.getPos();
    426                int[] ipos = pos.clone();
    427                while (it.hasNext()) {
    428                        dd.setSlice(a.getSliceView(it.getCurrentSlice()), ds);
    429                        for (int i = 0; i < size; i++) {
    430                                index[i] = i;
    431                        }
    432                        Arrays.sort(index, cmp);
    433
    434                        System.arraycopy(pos, 0, ipos, 0, pos.length);
    435                        for (int i = 0; i < size; i++) {
    436                                ipos[axis] = i;
    437                                id.set(index[i], ipos);
    438                        }
    439                }
    440
    441                return id;
    442        }
    443
    444        /**
    445         * Concatenate the set of datasets along given axis
    446         * @param as datasets
    447         * @param axis to concatenate along
    448         * @return concatenated dataset
    449         */
    450        public static Dataset concatenate(final IDataset[] as, int axis) {
    451                if (as == null || as.length == 0) {
    452                        utilsLogger.error("No datasets given");
    453                        throw new IllegalArgumentException("No datasets given");
    454                }
    455                IDataset a = as[0];
    456                if (as.length == 1) {
    457                        return convertToDataset(a.clone());
    458                }
    459
    460                int[] ashape = a.getShape();
    461                axis = ShapeUtils.checkAxis(ashape.length, axis);
    462                Class<? extends Dataset> ac = InterfaceUtils.getInterface(a);
    463                int anum = as.length;
    464                int isize = a.getElementsPerItem();
    465
    466                int i = 1;
    467                for (; i < anum; i++) {
    468                        if (!ac.equals(InterfaceUtils.getInterface(as[i]))) {
    469                                utilsLogger.error("Datasets are not of same type");
    470                                break;
    471                        }
    472                        if (!ShapeUtils.areShapesCompatible(ashape, as[i].getShape(), axis)) {
    473                                utilsLogger.error("Datasets' shapes are not equal");
    474                                break;
    475                        }
    476                        final int is = as[i].getElementsPerItem();
    477                        if (isize < is)
    478                                isize = is;
    479                }
    480                if (i < anum) {
    481                        utilsLogger.error("Dataset are not compatible");
    482                        throw new IllegalArgumentException("Datasets are not compatible");
    483                }
    484
    485                for (i = 1; i < anum; i++) {
    486                        ashape[axis] += as[i].getShape()[axis];
    487                }
    488
    489                Dataset result = DatasetFactory.zeros(isize, ac, ashape);
    490
    491                int[] start = new int[ashape.length];
    492                int[] stop = ashape;
    493                stop[axis] = 0;
    494                for (i = 0; i < anum; i++) {
    495                        IDataset b = as[i];
    496                        int[] bshape = b.getShape();
    497                        stop[axis] += bshape[axis];
    498                        result.setSlice(b, start, stop, null);
    499                        start[axis] += bshape[axis];
    500                }
    501
    502                return result;
    503        }
    504
    505        /**
    506         * Split a dataset into equal sections along given axis
    507         * @param a dataset
    508         * @param sections number of sections
    509         * @param axis to split along
    510         * @param checkEqual makes sure the division is into equal parts
    511         * @return list of split datasets
    512         */
    513        public static List<Dataset> split(final Dataset a, int sections, int axis, final boolean checkEqual) {
    514                int[] ashape = a.getShapeRef();
    515                axis = a.checkAxis(axis);
    516                int imax = ashape[axis];
    517                if (checkEqual && (imax%sections) != 0) {
    518                        utilsLogger.error("Number of sections does not divide axis into equal parts");
    519                        throw new IllegalArgumentException("Number of sections does not divide axis into equal parts");
    520                }
    521                int n = (imax + sections - 1) / sections;
    522                int[] indices = new int[sections-1];
    523                for (int i = 1; i < sections; i++)
    524                        indices[i-1] = n*i;
    525                return split(a, indices, axis);
    526        }
    527
    528        /**
    529         * Split a dataset into parts along given axis
    530         * @param a dataset
    531         * @param indices where to split
    532         * @param axis to split along
    533         * @return list of split datasets
    534         */
    535        public static List<Dataset> split(final Dataset a, int[] indices, int axis) {
    536                final int[] ashape = a.getShapeRef();
    537                axis = a.checkAxis(axis);
    538                final int rank = ashape.length;
    539                final int imax = ashape[axis];
    540
    541                final List<Dataset> result = new ArrayList<Dataset>();
    542
    543                final int[] nshape = ashape.clone();
    544                final int is = a.getElementsPerItem();
    545
    546                int oind = 0;
    547                final int[] start = new int[rank];
    548                final int[] stop = new int[rank];
    549                final int[] step = new int[rank];
    550                for (int i = 0; i < rank; i++) {
    551                        start[i] = 0;
    552                        stop[i] = ashape[i];
    553                        step[i] = 1;
    554                }
    555                for (int ind : indices) {
    556                        if (ind > imax) {
    557                                result.add(DatasetFactory.zeros(is, a.getClass(), 0));
    558                        } else {
    559                                nshape[axis] = ind - oind;
    560                                start[axis] = oind;
    561                                stop[axis] = ind;
    562                                Dataset n = DatasetFactory.zeros(is, a.getClass(), nshape);
    563                                IndexIterator iter = a.getSliceIterator(start, stop, step);
    564
    565                                a.fillDataset(n, iter);
    566                                result.add(n);
    567                                oind = ind;
    568                        }
    569                }
    570
    571                if (imax > oind) {
    572                        nshape[axis] = imax - oind;
    573                        start[axis] = oind;
    574                        stop[axis] = imax;
    575                        Dataset n = DatasetFactory.zeros(is, a.getClass(), nshape);
    576                        IndexIterator iter = a.getSliceIterator(start, stop, step);
    577
    578                        a.fillDataset(n, iter);
    579                        result.add(n);
    580                }
    581
    582                return result;
    583        }
    584
    585        /**
    586         * Constructs a dataset which has its elements along an axis replicated from
    587         * the original dataset by the number of times given in the repeats array.
    588         *
    589         * By default, axis=-1 implies using a flattened version of the input dataset
    590         *
    591         * @param <T> dataset class
    592         * @param a dataset
    593         * @param repeats number of repetitions
    594         * @param axis to repeat
    595         * @return dataset
    596         */
    597        public static <T extends Dataset> T repeat(T a, int[] repeats, int axis) {
    598                Serializable buf = a.getBuffer();
    599                int[] shape = a.getShape();
    600                int rank = shape.length;
    601
    602                if (axis >= rank) {
    603                        utilsLogger.warn("Axis value is out of bounds");
    604                        throw new IllegalArgumentException("Axis value is out of bounds");
    605                }
    606
    607                int alen;
    608                if (axis < 0) {
    609                        alen = a.getSize();
    610                        axis = 0;
    611                        rank = 1;
    612                        shape[0] = alen;
    613                } else {
    614                        alen = shape[axis];
    615                }
    616                int rlen = repeats.length;
    617                if (rlen != 1 && rlen != alen) {
    618                        utilsLogger.warn("Repeats array should have length of 1 or match chosen axis");
    619                        throw new IllegalArgumentException("Repeats array should have length of 1 or match chosen axis");
    620                }
    621
    622                for (int i = 0; i < rlen; i++) {
    623                        if (repeats[i] < 0) {
    624                                utilsLogger.warn("Negative repeat value is not allowed");
    625                                throw new IllegalArgumentException("Negative repeat value is not allowed");
    626                        }
    627                }
    628
    629                int[] newShape = new int[rank];
    630                for (int i = 0; i < rank; i ++)
    631                        newShape[i] = shape[i];
    632
    633                // do single repeat separately
    634                if (repeats.length == 1) {
    635                        newShape[axis] *= repeats[0];
    636                } else {
    637                        int nlen = 0;
    638                        for (int i = 0; i < alen; i++) {
    639                                nlen += repeats[i];
    640                        }
    641                        newShape[axis] = nlen;
    642                }
    643
    644                T rdata = DatasetFactory.zeros(a, newShape);
    645                Serializable nbuf = rdata.getBuffer();
    646
    647                int csize = a.getElementsPerItem(); // chunk size
    648                for (int i = axis+1; i < rank; i++) {
    649                        csize *= newShape[i];
    650                }
    651                int nout = 1;
    652                for (int i = 0; i < axis; i++) {
    653                        nout *= newShape[i];
    654                }
    655
    656                int oi = 0;
    657                int ni = 0;
    658                if (rlen == 1) { // do single repeat separately
    659                        for (int i = 0; i < nout; i++) {
    660                                for (int j = 0; j < shape[axis]; j++) {
    661                                        for (int k = 0; k < repeats[0]; k++) {
    662                                                System.arraycopy(buf, oi, nbuf, ni, csize);
    663                                                ni += csize;
    664                                        }
    665                                        oi += csize;
    666                                }
    667                        }
    668                } else {
    669                        for (int i = 0; i < nout; i++) {
    670                                for (int j = 0; j < shape[axis]; j++) {
    671                                        for (int k = 0; k < repeats[j]; k++) {
    672                                                System.arraycopy(buf, oi, nbuf, ni, csize);
    673                                                ni += csize;
    674                                        }
    675                                        oi += csize;
    676                                }
    677                        }
    678                }
    679
    680                return rdata;
    681        }
    682
    683        /**
    684         * Resize a dataset
    685         * @param <T> dataset class
    686         * @param a dataset
    687         * @param shape output shape
    688         * @return new dataset with new shape and items that are truncated or repeated, as necessary
    689         */
    690        public static <T extends Dataset> T resize(final T a, final int... shape) {
    691                int size = a.getSize();
    692                T rdata = DatasetFactory.zeros(a, shape);
    693                IndexIterator it = rdata.getIterator();
    694                while (it.hasNext()) {
    695                        rdata.setObjectAbs(it.index, a.getObjectAbs(it.index % size));
    696                }
    697
    698                return rdata;
    699        }
    700
    701        /**
    702         * Copy and cast a dataset
    703         *
    704         * @param d
    705         *            The dataset to be copied
    706         * @param dtype dataset type
    707         * @return copied dataset of given type
    708         * @deprecated Please use the class-based methods in DatasetUtils,
    709         *             such as {@link #copy(Class, IDataset)}
    710         */
    711        @Deprecated
    712        public static Dataset copy(final IDataset d, final int dtype) {
    713                return copy(DTypeUtils.getInterface(dtype), d);
    714        }
    715
    716        /**
    717         * Cast a dataset
    718         *
    719         * @param <T> dataset sub-interface
    720         * @param clazz dataset sub-interface
    721         * @param d
    722         *            The dataset to be copied
    723         * @return dataset of given class (or same dataset if already of the right class)
    724         */
    725        @SuppressWarnings("unchecked")
    726        public static <T extends Dataset> T copy(Class<T> clazz, final IDataset d) {
    727                Dataset a = convertToDataset(d);
    728                Dataset c = null;
    729                try {
    730                        // copy across the data
    731                        if (BooleanDataset.class.isAssignableFrom(clazz)) {
    732                                c = new BooleanDataset(a);
    733                        } else if (ByteDataset.class.isAssignableFrom(clazz)) {
    734                                c = new ByteDataset(a);
    735                        } else if (ShortDataset.class.isAssignableFrom(clazz)) {
    736                                c = new ShortDataset(a);
    737                        } else if (IntegerDataset.class.isAssignableFrom(clazz)) {
    738                                c = new IntegerDataset(a);
    739                        } else if (LongDataset.class.isAssignableFrom(clazz)) {
    740                                c = new LongDataset(a);
    741                        } else if (RGBByteDataset.class.isAssignableFrom(clazz)) {
    742                                if (a instanceof CompoundDataset) {
    743                                        c = new RGBByteDataset((CompoundDataset) a);
    744                                } else {
    745                                        c = new RGBByteDataset(a);
    746                                }
    747                        } else if (CompoundByteDataset.class.isAssignableFrom(clazz)) {
    748                                if (a instanceof CompoundByteDataset) {
    749                                        c = new CompoundByteDataset((CompoundDataset) a);
    750                                } else {
    751                                        c = new CompoundByteDataset(a);
    752                                }
    753                        } else if (RGBDataset.class.isAssignableFrom(clazz)) {
    754                                if (a instanceof CompoundDataset) {
    755                                        c = new RGBDataset((CompoundDataset) a);
    756                                } else {
    757                                        c = new RGBDataset(a);
    758                                }
    759                        } else if (CompoundShortDataset.class.isAssignableFrom(clazz)) {
    760                                if (a instanceof CompoundDataset) {
    761                                        c = new CompoundShortDataset((CompoundDataset) a);
    762                                } else {
    763                                        c = new CompoundShortDataset(a);
    764                                }
    765                        } else if (CompoundIntegerDataset.class.isAssignableFrom(clazz)) {
    766                                if (a instanceof CompoundDataset) {
    767                                        c = new CompoundIntegerDataset((CompoundDataset) a);
    768                                } else {
    769                                        c = new CompoundIntegerDataset(a);
    770                                }
    771                        } else if (CompoundLongDataset.class.isAssignableFrom(clazz)) {
    772                                if (a instanceof CompoundDataset) {
    773                                        c = new CompoundLongDataset((CompoundDataset) a);
    774                                } else {
    775                                        c = new CompoundLongDataset(a);
    776                                }
    777                        } else if (FloatDataset.class.isAssignableFrom(clazz)) {
    778                                c = new FloatDataset(a);
    779                        } else if (DoubleDataset.class.isAssignableFrom(clazz)) {
    780                                c = new DoubleDataset(a);
    781                        } else if (ComplexFloatDataset.class.isAssignableFrom(clazz)) {
    782                                c = new ComplexFloatDataset(a);
    783                        } else if (ComplexDoubleDataset.class.isAssignableFrom(clazz)) {
    784                                c = new ComplexDoubleDataset(a);
    785                        } else if (CompoundFloatDataset.class.isAssignableFrom(clazz)) {
    786                                if (a instanceof CompoundDataset) {
    787                                        c = new CompoundFloatDataset((CompoundDataset) a);
    788                                } else {
    789                                        c = new CompoundFloatDataset(a);
    790                                }
    791                        } else if (CompoundDoubleDataset.class.isAssignableFrom(clazz)) {
    792                                if (a instanceof CompoundDataset) {
    793                                        c = new CompoundDoubleDataset((CompoundDataset) a);
    794                                } else {
    795                                        c = new CompoundDoubleDataset(a);
    796                                }
    797                        } else if (DateDataset.class.isAssignableFrom(clazz)) {
    798                                c = DateDatasetImpl.createFromObject(a);
    799                        } else if (StringDataset.class.isAssignableFrom(clazz)) {
    800                                c = new StringDataset(a);
    801                        } else if (ObjectDataset.class.isAssignableFrom(clazz)) {
    802                                c = new ObjectDataset(a);
    803                        } else {
    804                                utilsLogger.error("Dataset of unknown type!");
    805                        }
    806                } catch (OutOfMemoryError e) {
    807                        utilsLogger.error("Not enough memory available to create dataset");
    808                        throw new OutOfMemoryError("Not enough memory available to create dataset");
    809                }
    810
    811                return (T) c;
    812        }
    813
    814        /**
    815         * Cast a dataset
    816         *
    817         * @param d
    818         *            The dataset to be cast
    819         * @param dtype dataset type
    820         * @return dataset of given type (or same dataset if already of the right type)
    821         * @deprecated Please use the class-based methods in DatasetUtils,
    822         *             such as {@link #cast(Class, IDataset)}
    823         */
    824        @Deprecated
    825        public static Dataset cast(final IDataset d, final int dtype) {
    826                return cast(DTypeUtils.getInterface(dtype), d);
    827        }
    828
    829        /**
    830         * Cast a dataset
    831         * @param <T> dataset sub-interface
    832         * @param clazz dataset sub-interface
    833         * @param d
    834         *            The dataset to be cast
    835         * @return dataset of given class
    836         */
    837        @SuppressWarnings("unchecked")
    838        public static <T extends Dataset> T cast(Class<T> clazz, final IDataset d) {
    839                Dataset a = convertToDataset(d);
    840
    841                if (a.getClass().equals(clazz)) {
    842                        return (T) a;
    843                }
    844
    845                if (a instanceof CompoundDataset) {
    846                        if (RGBByteDataset.class.isAssignableFrom(clazz)) {
    847                                return (T) RGBByteDataset.createFromCompoundDataset((CompoundDataset) a);
    848                        }
    849                        if (RGBDataset.class.isAssignableFrom(clazz)) {
    850                                return (T) RGBDataset.createFromCompoundDataset((CompoundDataset) a);
    851                        }
    852                }
    853
    854                return copy(clazz, d);
    855        }
    856
    857        /**
    858         * Cast a dataset
    859         *
    860         * @param d
    861         *            The dataset to be cast
    862         * @param repeat repeat elements over item
    863         * @param dtype dataset type
    864         * @param isize item size
    865         * @return dataset of given type
    866         * @deprecated Please use the class-based methods in DatasetUtils,
    867         *             such as {@link #cast(Class, IDataset)}
    868         */
    869        @Deprecated
    870        public static Dataset cast(final IDataset d, final boolean repeat, final int dtype, final int isize) {
    871                return cast(isize, DTypeUtils.getInterface(dtype), d, repeat);
    872        }
    873
    874        /**
    875         * Cast a dataset
    876         *
    877         * @param <T> dataset sub-interface
    878         * @param isize item size
    879         * @param clazz dataset sub-interface
    880         * @param d
    881         *            The dataset to be cast.
    882         * @param repeat repeat elements over item
    883         * @return dataset of given class
    884         * @since 2.3
    885         */
    886        @SuppressWarnings("unchecked")
    887        public static <T extends Dataset> T cast(final int isize, Class<T> clazz, final IDataset d, final boolean repeat) {
    888                Dataset a = convertToDataset(d);
    889
    890                if (a.getClass().equals(clazz) && a.getElementsPerItem() == isize) {
    891                        return (T) a;
    892                }
    893                if (isize <= 0) {
    894                        utilsLogger.error("Item size is invalid (>0)");
    895                        throw new IllegalArgumentException("Item size is invalid (>0)");
    896                }
    897                if (isize > 1 && !InterfaceUtils.isComplex(clazz) && InterfaceUtils.isElemental(clazz)) {
    898                        utilsLogger.error("Item size is inconsistent with dataset type");
    899                        throw new IllegalArgumentException("Item size is inconsistent with dataset type");
    900                }
    901
    902                Dataset c = null;
    903
    904                try {
    905                        // copy across the data
    906                        if (BooleanDataset.class.isAssignableFrom(clazz)) {
    907                                c = new BooleanDataset(a);
    908                        } else if (ByteDataset.class.isAssignableFrom(clazz)) {
    909                                c = new ByteDataset(a);
    910                        } else if (ShortDataset.class.isAssignableFrom(clazz)) {
    911                                c = new ShortDataset(a);
    912                        } else if (IntegerDataset.class.isAssignableFrom(clazz)) {
    913                                c = new IntegerDataset(a);
    914                        } else if (LongDataset.class.isAssignableFrom(clazz)) {
    915                                c = new LongDataset(a);
    916                        } else if (RGBByteDataset.class.isAssignableFrom(clazz)) {
    917                                if (a instanceof CompoundDataset) {
    918                                        c = RGBByteDataset.createFromCompoundDataset((CompoundDataset) a);
    919                                } else {
    920                                        c = new RGBByteDataset(a);
    921                                }
    922                        } else if (CompoundByteDataset.class.isAssignableFrom(clazz)) {
    923                                c = new CompoundByteDataset(isize, repeat, a);
    924                        } else if (RGBDataset.class.isAssignableFrom(clazz)) {
    925                                if (a instanceof CompoundDataset) {
    926                                        c = RGBDataset.createFromCompoundDataset((CompoundDataset) a);
    927                                } else {
    928                                        c = new RGBDataset(a);
    929                                }
    930                        } else if (CompoundShortDataset.class.isAssignableFrom(clazz)) {
    931                                c = new CompoundShortDataset(isize, repeat, a);
    932                        } else if (CompoundIntegerDataset.class.isAssignableFrom(clazz)) {
    933                                c = new CompoundIntegerDataset(isize, repeat, a);
    934                        } else if (CompoundLongDataset.class.isAssignableFrom(clazz)) {
    935                                c = new CompoundLongDataset(isize, repeat, a);
    936                        } else if (FloatDataset.class.isAssignableFrom(clazz)) {
    937                                c = new FloatDataset(a);
    938                        } else if (DoubleDataset.class.isAssignableFrom(clazz)) {
    939                                c = new DoubleDataset(a);
    940                        } else if (ComplexFloatDataset.class.isAssignableFrom(clazz)) {
    941                                c = new ComplexFloatDataset(a);
    942                        } else if (ComplexDoubleDataset.class.isAssignableFrom(clazz)) {
    943                                c = new ComplexDoubleDataset(a);
    944                        } else if (CompoundFloatDataset.class.isAssignableFrom(clazz)) {
    945                                c = new CompoundFloatDataset(isize, repeat, a);
    946                        } else if (CompoundDoubleDataset.class.isAssignableFrom(clazz)) {
    947                                c = new CompoundDoubleDataset(isize, repeat, a);
    948                        } else if (DateDataset.class.isAssignableFrom(clazz)) {
    949                                c = DateDatasetImpl.createFromObject(a);
    950                        } else if (StringDataset.class.isAssignableFrom(clazz)) {
    951                                c = new StringDataset(a);
    952                        } else if (ObjectDataset.class.isAssignableFrom(clazz)) {
    953                                c = new ObjectDataset(a);
    954                        } else {
    955                                utilsLogger.error("Dataset of unknown type!");
    956                        }
    957                } catch (OutOfMemoryError e) {
    958                        utilsLogger.error("Not enough memory available to create dataset");
    959                        throw new OutOfMemoryError("Not enough memory available to create dataset");
    960                }
    961
    962                return (T) c;
    963        }
    964
    965        /**
    966         * Cast array of datasets to a compound dataset
    967         *
    968         * @param <T> compound dataset sub-interface
    969         * @param clazz compound dataset sub-interface
    970         * @param a
    971         *            The datasets to be cast.
    972         * @return dataset of given class
    973         * @since 2.3
    974         */
    975        public static  <T extends CompoundDataset> T compoundCast(Class<T> clazz, final Dataset... a) {
    976                return createCompoundDataset(clazz, a);
    977        }
    978
    979
    980        /**
    981         * Cast array of datasets to a compound dataset
    982         *
    983         * @param clazz dataset class
    984         * @param a
    985         *            The datasets to be cast.
    986         * @return compound dataset of given class
    987         * @since 2.3
    988         */
    989        public static  CompoundDataset cast(Class<? extends Dataset> clazz, final Dataset... a) {
    990                return compoundCast(InterfaceUtils.getCompoundInterface(clazz), a);
    991        }
    992
    993        /**
    994         * Cast array of datasets to a compound dataset
    995         *
    996         * @param a
    997         *            The datasets to be cast.
    998         * @param dtype dataset type
    999         * @return compound dataset of given type
    1000         * @deprecated Please use the class-based methods in DatasetUtils,
    1001         *             such as {@link #cast(Class, Dataset...)}
    1002         */
    1003        @Deprecated
    1004        public static CompoundDataset cast(final Dataset[] a, final int dtype) {
    1005                return cast(DTypeUtils.getInterface(dtype), a);
    1006        }
    1007
    1008        /**
    1009         * Make a dataset unsigned by promoting it to a wider dataset type and unwrapping the signs
    1010         * of its contents
    1011         * @param a dataset
    1012         * @return unsigned dataset or original if it is not an integer dataset
    1013         */
    1014        public static Dataset makeUnsigned(IDataset a) {
    1015                return makeUnsigned(a, false);
    1016        }
    1017
    1018        /**
    1019         * Make a dataset unsigned by promoting it to a wider dataset type and unwrapping the signs
    1020         * of its contents
    1021         * @param a dataset
    1022         * @param check if true, then check for negative values
    1023         * @return unsigned dataset or original if it is not an integer dataset or it has been check for negative numbers
    1024         * @since 2.1
    1025         */
    1026        public static Dataset makeUnsigned(IDataset a, boolean check) {
    1027                Dataset d = convertToDataset(a);
    1028
    1029                if (d.hasFloatingPointElements()) {
    1030                        return d;
    1031                }
    1032                if (check && d.min(true).longValue() >= 0) {
    1033                        return d;
    1034                }
    1035
    1036                if (d instanceof ByteDataset) {
    1037                        d = new ShortDataset(d);
    1038                        unwrapUnsigned(d, 8);
    1039                } else if (d instanceof ShortDataset) {
    1040                        d = new IntegerDataset(d);
    1041                        unwrapUnsigned(d, 16);
    1042                } else if (d instanceof IntegerDataset) {
    1043                        d = new LongDataset(d);
    1044                        unwrapUnsigned(d, 32);
    1045                } else if (d instanceof CompoundByteDataset) {
    1046                        d = new CompoundShortDataset(d);
    1047                        unwrapUnsigned(d, 8);
    1048                } else if (d instanceof CompoundShortDataset) {
    1049                        d = new CompoundIntegerDataset(d);
    1050                        unwrapUnsigned(d, 16);
    1051                } else if (d instanceof CompoundIntegerDataset) {
    1052                        d = new CompoundLongDataset(d);
    1053                        unwrapUnsigned(d, 32);
    1054                }
    1055                return d;
    1056        }
    1057
    1058        /**
    1059         * Unwrap dataset elements so that all elements are unsigned
    1060         * @param a dataset
    1061         * @param bitWidth width of original primitive in bits
    1062         */
    1063        public static void unwrapUnsigned(Dataset a, final int bitWidth) {
    1064                final double dv = 1L << bitWidth;
    1065                final int isize = a.getElementsPerItem();
    1066                IndexIterator it = a.getIterator();
    1067
    1068                if (a instanceof ShortDataset) {
    1069                        ShortDataset sds = (ShortDataset) a;
    1070                        final short soffset = (short) dv;
    1071                        while (it.hasNext()) {
    1072                                final short x = sds.getAbs(it.index);
    1073                                if (x < 0) {
    1074                                        sds.setAbs(it.index, (short) (x + soffset));
    1075                                }
    1076                        }
    1077                } else if (a instanceof IntegerDataset) {
    1078                        IntegerDataset ids = (IntegerDataset) a;
    1079                        final int ioffset = (int) dv;
    1080                        while (it.hasNext()) {
    1081                                final int x = ids.getAbs(it.index);
    1082                                if (x < 0) {
    1083                                        ids.setAbs(it.index, x + ioffset);
    1084                                }
    1085                        }
    1086                } else if (a instanceof LongDataset) {
    1087                        LongDataset lds = (LongDataset) a;
    1088                        final long loffset = (long) dv;
    1089                        while (it.hasNext()) {
    1090                                final long x = lds.getAbs(it.index);
    1091                                if (x < 0) {
    1092                                        lds.setAbs(it.index, x + loffset);
    1093                                }
    1094                        }
    1095                } else if (a instanceof CompoundShortDataset) {
    1096                        CompoundShortDataset csds = (CompoundShortDataset) a;
    1097                        final short csoffset = (short) dv;
    1098                        final short[] csa = new short[isize];
    1099                        while (it.hasNext()) {
    1100                                csds.getAbs(it.index, csa);
    1101                                boolean dirty = false;
    1102                                for (int i = 0; i < isize; i++) {
    1103                                        short x = csa[i];
    1104                                        if (x < 0) {
    1105                                                csa[i] = (short) (x + csoffset);
    1106                                                dirty = true;
    1107                                        }
    1108                                }
    1109                                if (dirty) {
    1110                                        csds.setAbs(it.index, csa);
    1111                                }
    1112                        }
    1113                } else if (a instanceof CompoundIntegerDataset) {
    1114                        CompoundIntegerDataset cids = (CompoundIntegerDataset) a;
    1115                        final int cioffset = (int) dv;
    1116                        final int[] cia = new int[isize];
    1117                        while (it.hasNext()) {
    1118                                cids.getAbs(it.index, cia);
    1119                                boolean dirty = false;
    1120                                for (int i = 0; i < isize; i++) {
    1121                                        int x = cia[i];
    1122                                        if (x < 0) {
    1123                                                cia[i] = x + cioffset;
    1124                                                dirty = true;
    1125                                        }
    1126                                }
    1127                                if (dirty) {
    1128                                        cids.setAbs(it.index, cia);
    1129                                }
    1130                        }
    1131                } else if (a instanceof CompoundLongDataset) {
    1132                        CompoundLongDataset clds = (CompoundLongDataset) a;
    1133                        final long cloffset = (long) dv;
    1134                        final long[] cla = new long[isize];
    1135                        while (it.hasNext()) {
    1136                                clds.getAbs(it.index, cla);
    1137                                boolean dirty = false;
    1138                                for (int i = 0; i < isize; i++) {
    1139                                        long x = cla[i];
    1140                                        if (x < 0) {
    1141                                                cla[i] = x + cloffset;
    1142                                                dirty = true;
    1143                                        }
    1144                                }
    1145                                if (dirty) {
    1146                                        clds.setAbs(it.index, cla);
    1147                                }
    1148                        }
    1149                }
    1150        }
    1151
    1152        /**
    1153         * @param <T> dataset sub-interface
    1154         * @param clazz dataset sub-interface
    1155         * @param rows number of rows
    1156         * @param cols number of columns
    1157         * @param offset row offset
    1158         * @return a new 2d dataset of given shape and class, filled with ones on the (offset) diagonal
    1159         * @since 2.3
    1160         */
    1161        public static <T extends Dataset> T eye(final Class<T> clazz, final int rows, final int cols, final int offset) {
    1162                int[] shape = new int[] {rows, cols};
    1163                T a = DatasetFactory.zeros(clazz, shape);
    1164
    1165                int[] pos = new int[] {0, offset};
    1166                while (pos[1] < 0) {
    1167                        pos[0]++;
    1168                        pos[1]++;
    1169                }
    1170                while (pos[0] < rows && pos[1] < cols) {
    1171                        a.set(1, pos);
    1172                        pos[0]++;
    1173                        pos[1]++;
    1174                }
    1175
    1176                return a;
    1177        }
    1178
    1179        /**
    1180         * @param rows number of rows
    1181         * @param cols number of columns
    1182         * @param offset row offset
    1183         * @param dtype dataset type
    1184         * @return a new 2d dataset of given shape and type, filled with ones on the (offset) diagonal
    1185         * @deprecated Please use the class-based methods in DatasetUtils,
    1186         *             such as {@link #eye(Class, int, int, int)}
    1187         */
    1188        @Deprecated
    1189        public static Dataset eye(final int rows, final int cols, final int offset, final int dtype) {
    1190                return eye(DTypeUtils.getInterface(dtype), rows, cols, offset);
    1191        }
    1192
    1193        /**
    1194         * Create a (off-)diagonal matrix from items in dataset
    1195         *
    1196         * @param <T> dataset class
    1197         * @param a dataset
    1198         * @param offset distance right of diagonal
    1199         * @return diagonal matrix
    1200         */
    1201        public static <T extends Dataset> T diag(final T a, final int offset) {
    1202                final int rank = a.getRank();
    1203
    1204                if (rank == 0 || rank > 2) {
    1205                        utilsLogger.error("Rank of dataset should be one or two");
    1206                        throw new IllegalArgumentException("Rank of dataset should be one or two");
    1207                }
    1208
    1209                T result;
    1210                final int[] shape = a.getShapeRef();
    1211                if (rank == 1) {
    1212                        int side = shape[0] + Math.abs(offset);
    1213                        int[] pos = new int[] {side, side};
    1214                        result = DatasetFactory.zeros(a, pos);
    1215                        if (offset >= 0) {
    1216                                pos[0] = 0;
    1217                                pos[1] = offset;
    1218                        } else {
    1219                                pos[0] = -offset;
    1220                                pos[1] = 0;
    1221                        }
    1222                        int i = 0;
    1223                        while (pos[0] < side && pos[1] < side) {
    1224                                result.set(a.getObject(i++), pos);
    1225                                pos[0]++;
    1226                                pos[1]++;
    1227                        }
    1228                } else {
    1229                        int side = offset >= 0 ? Math.min(shape[0], shape[1]-offset) : Math.min(shape[0]+offset, shape[1]);
    1230                        if (side < 0)
    1231                                side = 0;
    1232                        result = DatasetFactory.zeros(a, side, side);
    1233
    1234                        if (side > 0) {
    1235                                int[] pos = offset >= 0 ? new int[] { 0, offset } : new int[] { -offset, 0 };
    1236                                int i = 0;
    1237                                while (pos[0] < shape[0] && pos[1] < shape[1]) {
    1238                                        result.set(a.getObject(pos), i++);
    1239                                        pos[0]++;
    1240                                        pos[1]++;
    1241                                }
    1242                        }
    1243                }
    1244
    1245                return (T) result;
    1246        }
    1247
    1248        /**
    1249         * Slice (or fully load), if necessary, a lazy dataset, otherwise take a slice view and
    1250         * convert to our dataset implementation. If a slice is necessary, this may cause resource
    1251         * problems when used on large datasets and throw runtime exceptions
    1252         * @param lazy can be null
    1253         * @return Converted dataset or null
    1254         * @throws DatasetException when cannot retrieve data
    1255         */
    1256        public static Dataset sliceAndConvertLazyDataset(ILazyDataset lazy) throws DatasetException {
    1257                if (lazy == null)
    1258                        return null;
    1259
    1260                IDataset data = lazy instanceof IDataset ? (IDataset) lazy.getSliceView() : lazy.getSlice();
    1261
    1262                return convertToDataset(data);
    1263        }
    1264
    1265        /**
    1266         * Convert (if necessary) a dataset obeying the interface to our implementation
    1267         * @param data can be null
    1268         * @return Converted dataset or null
    1269         */
    1270        public static Dataset convertToDataset(IDataset data) {
    1271                if (data == null)
    1272                        return null;
    1273
    1274                if (data instanceof Dataset) {
    1275                        return (Dataset) data;
    1276                }
    1277
    1278                final int isize = data.getElementsPerItem();
    1279                Class<? extends Dataset> clazz = InterfaceUtils.getInterfaceFromClass(isize, data.getElementClass());
    1280                if (isize <= 0) {
    1281                        throw new IllegalArgumentException("Datasets with " + isize + " elements per item not supported");
    1282                }
    1283
    1284                final Dataset result = DatasetFactory.zeros(isize, clazz, data.getShape());
    1285                result.setName(data.getName());
    1286
    1287                final IndexIterator it = result.getIterator(true);
    1288                final int[] pos = it.getPos();
    1289                if (BooleanDataset.class.isAssignableFrom(clazz)) {
    1290                        while (it.hasNext()) {
    1291                                result.setObjectAbs(it.index, data.getBoolean(pos));
    1292                        }
    1293                } else if (ByteDataset.class.isAssignableFrom(clazz)) {
    1294                        while (it.hasNext()) {
    1295                                result.setObjectAbs(it.index, data.getByte(pos));
    1296                        }
    1297                } else if (ShortDataset.class.isAssignableFrom(clazz)) {
    1298                        while (it.hasNext()) {
    1299                                result.setObjectAbs(it.index, data.getShort(pos));
    1300                        }
    1301                } else if (IntegerDataset.class.isAssignableFrom(clazz)) {
    1302                        while (it.hasNext()) {
    1303                                result.setObjectAbs(it.index, data.getInt(pos));
    1304                        }
    1305                } else if (LongDataset.class.isAssignableFrom(clazz)) {
    1306                        while (it.hasNext()) {
    1307                                result.setObjectAbs(it.index, data.getLong(pos));
    1308                        }
    1309                } else if (FloatDataset.class.isAssignableFrom(clazz)) {
    1310                        while (it.hasNext()) {
    1311                                result.setObjectAbs(it.index, data.getFloat(pos));
    1312                        }
    1313                } else if (DoubleDataset.class.isAssignableFrom(clazz)) {
    1314                        while (it.hasNext()) {
    1315                                result.setObjectAbs(it.index, data.getDouble(pos));
    1316                        }
    1317                } else {
    1318                        while (it.hasNext()) {
    1319                                result.setObjectAbs(it.index, data.getObject(pos));
    1320                        }
    1321                }
    1322
    1323                result.setErrors(data.getErrors());
    1324                return result;
    1325        }
    1326
    1327        /**
    1328         * Create a compound dataset from given datasets
    1329         * @param datasets inputs
    1330         * @return compound dataset or null if none given
    1331         */
    1332        public static CompoundDataset createCompoundDataset(final Dataset... datasets) {
    1333                if (datasets == null || datasets.length == 0)
    1334                        return null;
    1335
    1336                return (CompoundDataset) createCompoundDataset(InterfaceUtils.getCompoundInterface(datasets[0].getClass()), datasets);
    1337        }
    1338
    1339        /**
    1340         * Create a compound dataset from copying given datasets
    1341         * @param <T> compound dataset sub-interface
    1342         * @param clazz compound dataset sub-interface
    1343         * @param datasets inputs
    1344         * @return compound dataset or null if none given
    1345         */
    1346        @SuppressWarnings("unchecked")
    1347        public static <T extends CompoundDataset> T createCompoundDataset(Class<T> clazz, final Dataset... datasets) {
    1348                if (datasets == null || datasets.length == 0)
    1349                        return null;
    1350
    1351                CompoundDataset c = null;
    1352                if (RGBByteDataset.class.isAssignableFrom(clazz)) {
    1353                        if (datasets.length == 1) {
    1354                                c = new RGBByteDataset(datasets[0]);
    1355                        } else if (datasets.length == 3) {
    1356                                c = new RGBByteDataset(datasets[0], datasets[1], datasets[2]);
    1357                        } else {
    1358                                throw new IllegalArgumentException("Need one or three datasets for RGB dataset");
    1359                        }
    1360                } else if (CompoundByteDataset.class.isAssignableFrom(clazz)) {
    1361                        c = new CompoundByteDataset(datasets);
    1362                } else if (RGBDataset.class.isAssignableFrom(clazz)) {
    1363                        if (datasets.length == 1) {
    1364                                c = new RGBDataset(datasets[0]);
    1365                        } else if (datasets.length == 3) {
    1366                                c = new RGBDataset(datasets[0], datasets[1], datasets[2]);
    1367                        } else {
    1368                                throw new IllegalArgumentException("Need one or three datasets for RGB dataset");
    1369                        }
    1370                } else if (CompoundShortDataset.class.isAssignableFrom(clazz)) {
    1371                        c = new CompoundShortDataset(datasets);
    1372                } else if (CompoundIntegerDataset.class.isAssignableFrom(clazz)) {
    1373                        c = new CompoundIntegerDataset(datasets);
    1374                } else if (CompoundLongDataset.class.isAssignableFrom(clazz)) {
    1375                        c = new CompoundLongDataset(datasets);
    1376                } else if (ComplexFloatDataset.class.isAssignableFrom(clazz)) {
    1377                        if (datasets.length == 1) {
    1378                                c = new ComplexFloatDataset(datasets[0]);
    1379                        } else if (datasets.length >= 2) {
    1380                                c = new ComplexFloatDataset(datasets[0], datasets[1]);
    1381                        } else {
    1382                                throw new IllegalArgumentException("Need one or more datasets for complex dataset type");
    1383                        }
    1384                } else if (ComplexDoubleDataset.class.isAssignableFrom(clazz)) {
    1385                        if (datasets.length == 1) {
    1386                                c = new ComplexDoubleDataset(datasets[0]);
    1387                        } else if (datasets.length >= 2) {
    1388                                c = new ComplexDoubleDataset(datasets[0], datasets[1]);
    1389                        } else {
    1390                                throw new IllegalArgumentException("Need one or more datasets for complex dataset type");
    1391                        }
    1392                } else if (CompoundFloatDataset.class.isAssignableFrom(clazz)) {
    1393                        c = new CompoundFloatDataset(datasets);
    1394                } else if (DoubleDataset.class.equals(clazz) || CompoundDoubleDataset.class.isAssignableFrom(clazz)) {
    1395                        c = new CompoundDoubleDataset(datasets);
    1396                } else {
    1397                        utilsLogger.error("Dataset of unsupported interface");
    1398                }
    1399                return (T) c;
    1400        }
    1401
    1402        /**
    1403         * Create a compound dataset from given datasets
    1404         * @param dtype dataset type
    1405         * @param datasets for each element
    1406         * @return compound dataset or null if none given
    1407         * @deprecated Please use the class-based methods in DatasetUtils,
    1408         *             such as {@link #createCompoundDataset(Class, Dataset...)}
    1409         */
    1410        @Deprecated
    1411        public static CompoundDataset createCompoundDataset(final int dtype, final Dataset... datasets) {
    1412                return createCompoundDataset(InterfaceUtils.getCompoundInterface(DTypeUtils.getInterface(dtype)), datasets);
    1413        }
    1414
    1415        /**
    1416         * Create a compound dataset from given dataset, sharing data
    1417         * @param dataset input
    1418         * @param itemSize item size
    1419         * @return compound dataset
    1420         */
    1421        public static CompoundDataset createCompoundDataset(final Dataset dataset, final int itemSize) {
    1422                int[] shape = dataset.getShapeRef();
    1423                int[] nshape = shape;
    1424                if (shape != null && itemSize > 1) {
    1425                        int size = ShapeUtils.calcSize(shape);
    1426                        if (size % itemSize != 0) {
    1427                                throw new IllegalArgumentException("Input dataset has number of items that is not a multiple of itemSize");
    1428                        }
    1429                        int d = shape.length;
    1430                        int l = 1;
    1431                        while (--d >= 0) {
    1432                                l *= shape[d];
    1433                                if (l % itemSize == 0) {
    1434                                        break;
    1435                                }
    1436                        }
    1437                        assert d >= 0;
    1438                        nshape = new int[d + 1];
    1439                        for (int i = 0; i < d; i++) {
    1440                                nshape[i] = shape[i];
    1441                        }
    1442                        nshape[d] = l / itemSize;
    1443                }
    1444
    1445                if (dataset instanceof ByteDataset) {
    1446                        return new CompoundByteDataset(itemSize, (byte[]) dataset.getBuffer(), nshape);
    1447                } else if (dataset instanceof ShortDataset) {
    1448                        return new CompoundShortDataset(itemSize, (short[]) dataset.getBuffer(), nshape);
    1449                } else if (dataset instanceof IntegerDataset) {
    1450                        return new CompoundIntegerDataset(itemSize, (int[]) dataset.getBuffer(), nshape);
    1451                } else if (dataset instanceof LongDataset) {
    1452                        return new CompoundLongDataset(itemSize, (long[]) dataset.getBuffer(), nshape);
    1453                } else if (dataset instanceof FloatDataset) {
    1454                        return new CompoundFloatDataset(itemSize, (float[]) dataset.getBuffer(), nshape);
    1455                } else if (dataset instanceof DoubleDataset) {
    1456                        return new CompoundDoubleDataset(itemSize, (double[]) dataset.getBuffer(), nshape);
    1457                }
    1458
    1459                utilsLogger.error("Dataset interface not supported for this operation");
    1460                throw new UnsupportedOperationException("Dataset interface not supported");
    1461        }
    1462
    1463
    1464        /**
    1465         * Create a compound dataset by using last axis as elements of an item
    1466         * @param a dataset
    1467         * @param shareData if true, then share data
    1468         * @return compound dataset
    1469         */
    1470        public static CompoundDataset createCompoundDatasetFromLastAxis(final Dataset a, final boolean shareData) {
    1471                if (a instanceof ByteDataset) {
    1472                        return CompoundByteDataset.createCompoundDatasetWithLastDimension(a, shareData);
    1473                } else if (a instanceof ShortDataset) {
    1474                        return CompoundShortDataset.createCompoundDatasetWithLastDimension(a, shareData);
    1475                } else if (a instanceof IntegerDataset) {
    1476                        return CompoundIntegerDataset.createCompoundDatasetWithLastDimension(a, shareData);
    1477                } else if (a instanceof LongDataset) {
    1478                        return CompoundLongDataset.createCompoundDatasetWithLastDimension(a, shareData);
    1479                } else if (a instanceof FloatDataset) {
    1480                        return CompoundFloatDataset.createCompoundDatasetWithLastDimension(a, shareData);
    1481                } else if (a instanceof DoubleDataset) {
    1482                        return CompoundDoubleDataset.createCompoundDatasetWithLastDimension(a, shareData);
    1483                }
    1484
    1485                utilsLogger.error("Dataset interface not supported for this operation");
    1486                throw new UnsupportedOperationException("Dataset interface not supported");
    1487        }
    1488
    1489        /**
    1490         * Create a dataset from a compound dataset by using elements of an item as last axis
    1491         * <p>
    1492         * In the case where the number of elements is one, the last axis is squeezed out.
    1493         * @param a dataset
    1494         * @param shareData if true, then share data
    1495         * @return non-compound dataset
    1496         */
    1497        public static Dataset createDatasetFromCompoundDataset(final CompoundDataset a, final boolean shareData) {
    1498                return a.asNonCompoundDataset(shareData);
    1499        }
    1500
    1501        /**
    1502         * Create a copy that has been coerced to an appropriate dataset type
    1503         * depending on the input object's class
    1504         *
    1505         * @param a dataset
    1506         * @param obj input object
    1507         * @return coerced copy of dataset
    1508         */
    1509        public static Dataset coerce(Dataset a, Object obj) {
    1510                return cast(InterfaceUtils.getBestInterface(a.getClass(), InterfaceUtils.getInterface(obj)), a.clone());
    1511        }
    1512
    1513        /**
    1514         * Function that returns a normalised dataset which is bounded between 0 and 1
    1515         * @param a dataset
    1516         * @return normalised dataset
    1517         */
    1518        public static Dataset norm(Dataset a) {
    1519                double amin = a.min().doubleValue();
    1520                double aptp = a.max().doubleValue() - amin;
    1521                Dataset temp = Maths.subtract(a, amin);
    1522                temp.idivide(aptp);
    1523                return temp;
    1524        }
    1525
    1526        /**
    1527         * Function that returns a normalised compound dataset which is bounded between 0 and 1. There
    1528         * are (at least) two ways to normalise a compound dataset: per element - extrema for each element
    1529         * in a compound item is used, i.e. many min/max pairs; over all elements - extrema for all elements
    1530         * is used, i.e. one min/max pair.
    1531         * @param a dataset
    1532         * @param overAllElements if true, then normalise over all elements in each item
    1533         * @return normalised dataset
    1534         */
    1535        public static CompoundDataset norm(CompoundDataset a, boolean overAllElements) {
    1536                double[] amin = a.minItem();
    1537                double[] amax = a.maxItem();
    1538                final int is = a.getElementsPerItem();
    1539                Dataset result;
    1540
    1541                if (overAllElements) {
    1542                        Arrays.sort(amin);
    1543                        Arrays.sort(amax);
    1544                        double aptp = amax[0] - amin[0];
    1545
    1546                        result = Maths.subtract(a, amin[0]);
    1547                        result.idivide(aptp);
    1548                } else {
    1549                        double[] aptp = new double[is];
    1550                        for (int j = 0; j < is; j++) {
    1551                                aptp[j] = amax[j] - amin[j];
    1552                        }
    1553
    1554                        result = Maths.subtract(a, amin);
    1555                        result.idivide(aptp);
    1556                }
    1557                return (CompoundDataset) result;
    1558        }
    1559
    1560        /**
    1561         * Function that returns a normalised dataset which is bounded between 0 and 1
    1562         * and has been distributed on a log10 scale
    1563         * @param a dataset
    1564         * @return normalised dataset
    1565         */
    1566        public static Dataset lognorm(Dataset a) {
    1567                double amin = a.min().doubleValue();
    1568                double aptp = Math.log10(a.max().doubleValue() - amin + 1.);
    1569                Dataset temp = Maths.subtract(a, amin - 1.);
    1570                temp = Maths.log10(temp);
    1571                temp = Maths.divide(temp, aptp);
    1572                return temp;
    1573        }
    1574
    1575        /**
    1576         * Function that returns a normalised dataset which is bounded between 0 and 1
    1577         * and has been distributed on a natural log scale
    1578         * @param a dataset
    1579         * @return normalised dataset
    1580         */
    1581        public static Dataset lnnorm(Dataset a) {
    1582                double amin = a.min().doubleValue();
    1583                double aptp = Math.log(a.max().doubleValue() - amin + 1.);
    1584                Dataset temp = Maths.subtract(a, amin - 1.);
    1585                temp = Maths.log(temp);
    1586                temp = Maths.divide(temp, aptp);
    1587                return temp;
    1588        }
    1589
    1590        /**
    1591         * Construct a list of datasets where each represents a coordinate varying over the hypergrid
    1592         * formed by the input list of axes
    1593         *
    1594         * @param axes an array of 1D datasets representing axes
    1595         * @return a list of coordinate datasets
    1596         */
    1597        public static List<Dataset> meshGrid(final Dataset... axes) {
    1598                List<Dataset> result = new ArrayList<Dataset>();
    1599                int rank = axes.length;
    1600
    1601                if (rank < 2) {
    1602                        utilsLogger.error("Two or more axes datasets are required");
    1603                        throw new IllegalArgumentException("Two or more axes datasets are required");
    1604                }
    1605
    1606                int[] nshape = new int[rank];
    1607
    1608                for (int i = 0; i < rank; i++) {
    1609                        Dataset axis = axes[i];
    1610                        if (axis.getRank() != 1) {
    1611                                utilsLogger.error("Given axis is not 1D");
    1612                                throw new IllegalArgumentException("Given axis is not 1D");
    1613                        }
    1614                        nshape[i] = axis.getSize();
    1615                }
    1616
    1617                for (int i = 0; i < rank; i++) {
    1618                        Dataset axis = axes[i];
    1619                        Dataset coord = DatasetFactory.zeros(axis.getClass(), nshape);
    1620                        result.add(coord);
    1621
    1622                        final int alen = axis.getSize();
    1623                        for (int j = 0; j < alen; j++) {
    1624                                final Object obj = axis.getObjectAbs(j);
    1625                                PositionIterator pi = coord.getPositionIterator(i);
    1626                                final int[] pos = pi.getPos();
    1627
    1628                                pos[i] = j;
    1629                                while (pi.hasNext()) {
    1630                                        coord.set(obj, pos);
    1631                                }
    1632                        }
    1633                }
    1634
    1635                return result;
    1636        }
    1637
    1638        /**
    1639         * Generate an index dataset for given dataset shape where sub-datasets contain index values
    1640         *
    1641         * @param shape for indexing
    1642         * @return an index dataset
    1643         */
    1644        public static IntegerDataset indices(int... shape) {
    1645                // now create another dataset to plot against
    1646                final int rank = shape.length;
    1647                int[] nshape = new int[rank+1];
    1648                nshape[0] = rank;
    1649                for (int i = 0; i < rank; i++) {
    1650                        nshape[i+1] = shape[i];
    1651                }
    1652
    1653                IntegerDataset index = new IntegerDataset(nshape);
    1654
    1655                if (rank == 1) {
    1656                        final int alen = shape[0];
    1657                        int[] pos = new int[2];
    1658                        for (int j = 0; j < alen; j++) {
    1659                                pos[1] = j;
    1660                                index.set(j, pos);
    1661                        }
    1662                } else {
    1663                        for (int i = 1; i <= rank; i++) {
    1664                                final int alen = nshape[i];
    1665                                for (int j = 0; j < alen; j++) {
    1666                                        PositionIterator pi = index.getPositionIterator(0, i);
    1667                                        final int[] pos = pi.getPos();
    1668
    1669                                        pos[0] = i-1;
    1670                                        pos[i] = j;
    1671                                        while (pi.hasNext()) {
    1672                                                index.set(j, pos);
    1673                                        }
    1674                                }
    1675                        }
    1676                }
    1677                return index;
    1678        }
    1679
    1680        /**
    1681         * Get the centroid value of a dataset, this function works out the centroid in every direction
    1682         *
    1683         * @param a
    1684         *            the dataset to be analysed
    1685         * @param bases the optional array of base coordinates to use as weights.
    1686         * This defaults to the mid-point of indices
    1687         * @return a double array containing the centroid for each dimension
    1688         */
    1689        public static double[] centroid(Dataset a, Dataset... bases) {
    1690                int rank = a.getRank();
    1691                if (bases.length > 0 && bases.length != rank) {
    1692                        throw new IllegalArgumentException("Number of bases must be zero or match rank of dataset");
    1693                }
    1694
    1695                int[] shape = a.getShapeRef();
    1696                if (bases.length == rank) {
    1697                        for (int i = 0; i < rank; i++) {
    1698                                Dataset b = bases[i];
    1699                                if (b.getRank() != 1 && b.getSize() != shape[i]) {
    1700                                        throw new IllegalArgumentException("A base does not have shape to match given dataset");
    1701                                }
    1702                        }
    1703                }
    1704
    1705                double[] dc = new double[rank];
    1706                if (rank == 0)
    1707                        return dc;
    1708
    1709                final PositionIterator iter = new PositionIterator(shape);
    1710                final int[] pos = iter.getPos();
    1711
    1712                double tsum = 0.0;
    1713                while (iter.hasNext()) {
    1714                        double val = a.getDouble(pos);
    1715                        tsum += val;
    1716                        for (int d = 0; d < rank; d++) {
    1717                                Dataset b = bases.length == 0 ? null : bases[d];
    1718                                if (b == null) {
    1719                                        dc[d] += (pos[d] + 0.5) * val;
    1720                                } else {
    1721                                        dc[d] += b.getElementDoubleAbs(pos[d]) * val;
    1722                                }
    1723                        }
    1724                }
    1725
    1726                for (int d = 0; d < rank; d++) {
    1727                        dc[d] /= tsum;
    1728                }
    1729                return dc;
    1730        }
    1731
    1732        /**
    1733         * Find linearly-interpolated crossing points where the given dataset crosses the given value
    1734         *
    1735         * @param d dataset
    1736         * @param value crossing value
    1737         * @return list of interpolated indices
    1738         */
    1739        public static List<Double> crossings(Dataset d, double value) {
    1740                if (d.getRank() != 1) {
    1741                        utilsLogger.error("Only 1d datasets supported");
    1742                        throw new UnsupportedOperationException("Only 1d datasets supported");
    1743                }
    1744                List<Double> results = new ArrayList<Double>();
    1745
    1746                // run through all pairs of points on the line and see if value lies within
    1747                IndexIterator it = d.getIterator();
    1748                double y1, y2;
    1749
    1750                y2 = it.hasNext() ? d.getElementDoubleAbs(it.index) : 0;
    1751                double x = 1;
    1752                while (it.hasNext()) {
    1753                        y1 = y2;
    1754                        y2 = d.getElementDoubleAbs(it.index);
    1755                        // check if value lies within pair [y1, y2]
    1756                        if ((y1 <= value && value < y2) || (y1 > value && y2 <= value)) {
    1757                                final double f = (value - y2)/(y2 - y1); // negative distance from right to left
    1758                                results.add(x + f);
    1759                        }
    1760                        x++;
    1761                }
    1762                if (y2 == value) { // add end point of it intersects
    1763                        results.add(x);
    1764                }
    1765
    1766                return results;
    1767        }
    1768
    1769        /**
    1770         * Find x values of all the crossing points of the dataset with the given y value
    1771         *
    1772         * @param xAxis
    1773         *            Dataset of the X axis that needs to be looked at
    1774         * @param yAxis
    1775         *            Dataset of the Y axis that needs to be looked at
    1776         * @param yValue
    1777         *            The y value the X values are required for
    1778         * @return An list of doubles containing all the X coordinates of where the line crosses
    1779         */
    1780        public static List<Double> crossings(Dataset xAxis, Dataset yAxis, double yValue) {
    1781                if (xAxis.getSize() > yAxis.getSize()) {
    1782                        throw new IllegalArgumentException(
    1783                                        "Number of values of yAxis must as least be equal to the number of values of xAxis");
    1784                }
    1785
    1786                List<Double> results = new ArrayList<Double>();
    1787
    1788                List<Double> indices = crossings(yAxis, yValue);
    1789
    1790                for (double xi : indices) {
    1791                        results.add(Maths.interpolate(xAxis, xi));
    1792                }
    1793                return results;
    1794        }
    1795
    1796        /**
    1797         * Function that uses the crossings function but prunes the result, so that multiple crossings within a
    1798         * certain proportion of the overall range of the x values
    1799         *
    1800         * @param xAxis
    1801         *            Dataset of the X axis
    1802         * @param yAxis
    1803         *            Dataset of the Y axis
    1804         * @param yValue
    1805         *            The y value the x values are required for
    1806         * @param xRangeProportion
    1807         *            The proportion of the overall x spread used to prune result
    1808         * @return A list containing all the unique crossing points
    1809         */
    1810        public static List<Double> crossings(Dataset xAxis, Dataset yAxis, double yValue, double xRangeProportion) {
    1811                // get the values found
    1812                List<Double> vals = crossings(xAxis, yAxis, yValue);
    1813
    1814                // use the proportion to calculate the error spacing
    1815                double error = xRangeProportion * xAxis.peakToPeak().doubleValue();
    1816
    1817                int i = 0;
    1818                // now go through and check for groups of three crossings which are all
    1819                // within the boundaries
    1820                while (i <= vals.size() - 3) {
    1821                        double v1 = Math.abs(vals.get(i) - vals.get(i + 2));
    1822                        if (v1 < error) {
    1823                                // these 3 points should be treated as one
    1824                                // make the first point equal to the average of them all
    1825                                vals.set(i + 2, ((vals.get(i) + vals.get(i + 1) + vals.get(i + 2)) / 3.0));
    1826                                // remove the other offending points
    1827                                vals.remove(i);
    1828                                vals.remove(i);
    1829                        } else {
    1830                                i++;
    1831                        }
    1832                }
    1833
    1834                // once the thinning process has been completed, return the pruned list
    1835                return vals;
    1836        }
    1837
    1838        // recursive function
    1839        private static void setRow(Object row, Dataset a, int... pos) {
    1840                final int l = Array.getLength(row);
    1841                final int rank = pos.length;
    1842                final int[] npos = Arrays.copyOf(pos, rank+1);
    1843                Object r;
    1844                if (rank+1 < a.getRank()) {
    1845                        for (int i = 0; i < l; i++) {
    1846                                npos[rank] = i;
    1847                                r = Array.get(row, i);
    1848                                setRow(r, a, npos);
    1849                        }
    1850                } else {
    1851                        for (int i = 0; i < l; i++) {
    1852                                npos[rank] = i;
    1853                                r = a.getObject(npos);
    1854                                Array.set(row, i, r);
    1855                        }
    1856                }
    1857        }
    1858
    1859        /**
    1860         * Create Java array (of arrays) from dataset
    1861         * @param a dataset
    1862         * @return Java array (of arrays...)
    1863         */
    1864        public static Object createJavaArray(Dataset a) {
    1865                if (a.getElementsPerItem() > 1) {
    1866                        a = createDatasetFromCompoundDataset((CompoundDataset) a, true);
    1867                }
    1868                Object matrix;
    1869
    1870                int[] shape = a.getShapeRef();
    1871                if (a instanceof BooleanDataset) {
    1872                        matrix = Array.newInstance(boolean.class, shape);
    1873                } else if (a instanceof ByteDataset) {
    1874                        matrix = Array.newInstance(byte.class, shape);
    1875                } else if (a instanceof ShortDataset) {
    1876                        matrix = Array.newInstance(short.class, shape);
    1877                } else if (a instanceof IntegerDataset) {
    1878                        matrix = Array.newInstance(int.class, shape);
    1879                } else if (a instanceof LongDataset) {
    1880                        matrix = Array.newInstance(long.class, shape);
    1881                } else if (a instanceof FloatDataset) {
    1882                        matrix = Array.newInstance(float.class, shape);
    1883                } else if (a instanceof DoubleDataset) {
    1884                        matrix = Array.newInstance(double.class, shape);
    1885                } else {
    1886                        utilsLogger.error("Dataset type not supported");
    1887                        throw new IllegalArgumentException("Dataset type not supported");
    1888                }
    1889                // populate matrix
    1890                setRow(matrix, a);
    1891                return matrix;
    1892        }
    1893
    1894        /**
    1895         * Removes NaNs and infinities from floating point datasets.
    1896         * All other dataset types are ignored.
    1897         *
    1898         * @param a dataset
    1899         * @param value replacement value
    1900         */
    1901        public static void removeNansAndInfinities(Dataset a, final Number value) {
    1902                if (a instanceof DoubleDataset) {
    1903                        final double dvalue = DTypeUtils.toReal(value);
    1904                        final DoubleDataset set = (DoubleDataset) a;
    1905                        final IndexIterator it = set.getIterator();
    1906                        final double[] data = set.getData();
    1907                        while (it.hasNext()) {
    1908                                double x = data[it.index];
    1909                                if (Double.isNaN(x) || Double.isInfinite(x))
    1910                                        data[it.index] = dvalue;
    1911                        }
    1912                } else if (a instanceof FloatDataset) {
    1913                        final float fvalue = (float) DTypeUtils.toReal(value);
    1914                        final FloatDataset set = (FloatDataset) a;
    1915                        final IndexIterator it = set.getIterator();
    1916                        final float[] data = set.getData();
    1917                        while (it.hasNext()) {
    1918                                float x = data[it.index];
    1919                                if (Float.isNaN(x) || Float.isInfinite(x))
    1920                                        data[it.index] = fvalue;
    1921                        }
    1922                } else if (a instanceof CompoundDoubleDataset) {
    1923                        final double dvalue = DTypeUtils.toReal(value);
    1924                        final CompoundDoubleDataset set = (CompoundDoubleDataset) a;
    1925                        final int is = set.getElementsPerItem();
    1926                        final IndexIterator it = set.getIterator();
    1927                        final double[] data = set.getData();
    1928                        while (it.hasNext()) {
    1929                                for (int j = 0; j < is; j++) {
    1930                                        double x = data[it.index + j];
    1931                                        if (Double.isNaN(x) || Double.isInfinite(x))
    1932                                                data[it.index + j] = dvalue;
    1933                                }
    1934                        }
    1935                } else if (a instanceof CompoundFloatDataset) {
    1936                        final float fvalue = (float) DTypeUtils.toReal(value);
    1937                        final CompoundFloatDataset set = (CompoundFloatDataset) a;
    1938                        final int is = set.getElementsPerItem();
    1939                        final IndexIterator it = set.getIterator();
    1940                        final float[] data = set.getData();
    1941                        while (it.hasNext()) {
    1942                                for (int j = 0; j < is; j++) {
    1943                                        float x = data[it.index + j];
    1944                                        if (Float.isNaN(x) || Float.isInfinite(x))
    1945                                                data[it.index + j] = fvalue;
    1946                                }
    1947                        }
    1948                }
    1949        }
    1950
    1951        /**
    1952         * Make floating point datasets contain only finite values. Infinities and NaNs are replaced
    1953         * by +/- MAX_VALUE and 0, respectively.
    1954         * All other dataset types are ignored.
    1955         *
    1956         * @param a dataset
    1957         */
    1958        public static void makeFinite(Dataset a) {
    1959                if (a instanceof DoubleDataset) {
    1960                        final DoubleDataset set = (DoubleDataset) a;
    1961                        final IndexIterator it = set.getIterator();
    1962                        final double[] data = set.getData();
    1963                        while (it.hasNext()) {
    1964                                final double x = data[it.index];
    1965                                if (Double.isNaN(x))
    1966                                        data[it.index] = 0;
    1967                                else if (Double.isInfinite(x))
    1968                                        data[it.index] = x > 0 ? Double.MAX_VALUE : -Double.MAX_VALUE;
    1969                        }
    1970                } else if (a instanceof FloatDataset) {
    1971                        final FloatDataset set = (FloatDataset) a;
    1972                        final IndexIterator it = set.getIterator();
    1973                        final float[] data = set.getData();
    1974                        while (it.hasNext()) {
    1975                                final float x = data[it.index];
    1976                                if (Float.isNaN(x))
    1977                                        data[it.index] = 0;
    1978                                else if (Float.isInfinite(x))
    1979                                        data[it.index] = x > 0 ? Float.MAX_VALUE : -Float.MAX_VALUE;
    1980                        }
    1981                } else if (a instanceof CompoundDoubleDataset) {
    1982                        final CompoundDoubleDataset set = (CompoundDoubleDataset) a;
    1983                        final int is = set.getElementsPerItem();
    1984                        final IndexIterator it = set.getIterator();
    1985                        final double[] data = set.getData();
    1986                        while (it.hasNext()) {
    1987                                for (int j = 0; j < is; j++) {
    1988                                        final double x = data[it.index + j];
    1989                                        if (Double.isNaN(x))
    1990                                                data[it.index + j] = 0;
    1991                                        else if (Double.isInfinite(x))
    1992                                                data[it.index + j] = x > 0 ? Double.MAX_VALUE : -Double.MAX_VALUE;
    1993                                }
    1994                        }
    1995                } else if (a instanceof CompoundFloatDataset) {
    1996                        final CompoundFloatDataset set = (CompoundFloatDataset) a;
    1997                        final int is = set.getElementsPerItem();
    1998                        final IndexIterator it = set.getIterator();
    1999                        final float[] data = set.getData();
    2000                        while (it.hasNext()) {
    2001                                for (int j = 0; j < is; j++) {
    2002                                        final float x = data[it.index + j];
    2003                                        if (Float.isNaN(x))
    2004                                                data[it.index + j] = 0;
    2005                                        else if (Float.isInfinite(x))
    2006                                                data[it.index + j] = x > 0 ? Float.MAX_VALUE : -Float.MAX_VALUE;
    2007                                }
    2008                        }
    2009                }
    2010        }
    2011
    2012        /**
    2013         * Find absolute index of first value in dataset that is equal to given number
    2014         * @param a dataset
    2015         * @param n value
    2016         * @return absolute index (if greater than a.getSize() then no value found)
    2017         */
    2018        public static int findIndexEqualTo(final Dataset a, final double n) {
    2019                IndexIterator iter = a.getIterator();
    2020                while (iter.hasNext()) {
    2021                        if (a.getElementDoubleAbs(iter.index) == n)
    2022                                break;
    2023                }
    2024
    2025                return iter.index;
    2026        }
    2027
    2028        /**
    2029         * Find absolute index of first value in dataset that is greater than given number
    2030         * @param a dataset
    2031         * @param n value
    2032         * @return absolute index (if greater than a.getSize() then no value found)
    2033         */
    2034        public static int findIndexGreaterThan(final Dataset a, final double n) {
    2035                IndexIterator iter = a.getIterator();
    2036                while (iter.hasNext()) {
    2037                        if (a.getElementDoubleAbs(iter.index) > n)
    2038                                break;
    2039                }
    2040
    2041                return iter.index;
    2042        }
    2043
    2044        /**
    2045         * Find absolute index of first value in dataset that is greater than or equal to given number
    2046         * @param a dataset
    2047         * @param n value
    2048         * @return absolute index (if greater than a.getSize() then no value found)
    2049         */
    2050        public static int findIndexGreaterThanOrEqualTo(final Dataset a, final double n) {
    2051                IndexIterator iter = a.getIterator();
    2052                while (iter.hasNext()) {
    2053                        if (a.getElementDoubleAbs(iter.index) >= n)
    2054                                break;
    2055                }
    2056
    2057                return iter.index;
    2058        }
    2059
    2060        /**
    2061         * Find absolute index of first value in dataset that is less than given number
    2062         * @param a dataset
    2063         * @param n value
    2064         * @return absolute index (if greater than a.getSize() then no value found)
    2065         */
    2066        public static int findIndexLessThan(final Dataset a, final double n) {
    2067                IndexIterator iter = a.getIterator();
    2068                while (iter.hasNext()) {
    2069                        if (a.getElementDoubleAbs(iter.index) < n)
    2070                                break;
    2071                }
    2072
    2073                return iter.index;
    2074        }
    2075
    2076        /**
    2077         * Find absolute index of first value in dataset that is less than or equal to given number
    2078         * @param a dataset
    2079         * @param n value
    2080         * @return absolute index (if greater than a.getSize() then no value found)
    2081         */
    2082        public static int findIndexLessThanOrEqualTo(final Dataset a, final double n) {
    2083                IndexIterator iter = a.getIterator();
    2084                while (iter.hasNext()) {
    2085                        if (a.getElementDoubleAbs(iter.index) <= n)
    2086                                break;
    2087                }
    2088
    2089                return iter.index;
    2090        }
    2091
    2092        /**
    2093         * Find first occurrences in one dataset of values given in another sorted dataset
    2094         * @param a dataset
    2095         * @param values sorted 1D dataset of values to find
    2096         * @return absolute indexes of those first occurrences (-1 is used to indicate value not found)
    2097         */
    2098        public static IntegerDataset findFirstOccurrences(final Dataset a, final Dataset values) {
    2099                if (values.getRank() != 1) {
    2100                        throw new IllegalArgumentException("Values dataset must be 1D");
    2101                }
    2102                IntegerDataset indexes = new IntegerDataset(values.getSize());
    2103                indexes.fill(-1);
    2104
    2105                IndexIterator it = a.getIterator();
    2106                final int n = values.getSize();
    2107                if (values instanceof LongDataset) {
    2108                        while (it.hasNext()) {
    2109                                long x = a.getElementLongAbs(it.index);
    2110
    2111                                int l = 0; // binary search to find value in sorted dataset
    2112                                long vl = values.getLong(l);
    2113                                if (x <= vl) {
    2114                                        if (x == vl && indexes.getAbs(l) < 0)
    2115                                                indexes.setAbs(l, it.index);
    2116                                        continue;
    2117                                }
    2118                                int h = n - 1;
    2119                                long vh = values.getLong(h);
    2120                                if (x >= vh) {
    2121                                        if (x == vh && indexes.getAbs(h) < 0)
    2122                                                indexes.setAbs(h, it.index);
    2123                                        continue;
    2124                                }
    2125                                while (h - l > 1) {
    2126                                        int m = (l + h) / 2;
    2127                                        long vm = values.getLong(m);
    2128                                        if (x < vm) {
    2129                                                h = m;
    2130                                        } else if (x > vm) {
    2131                                                l = m;
    2132                                        } else {
    2133                                                if (indexes.getAbs(m) < 0)
    2134                                                        indexes.setAbs(m, it.index);
    2135                                                break;
    2136                                        }
    2137                                }
    2138                        }
    2139                } else {
    2140                        while (it.hasNext()) {
    2141                                double x = a.getElementDoubleAbs(it.index);
    2142
    2143                                int l = 0; // binary search to find value in sorted dataset
    2144                                double vl = values.getDouble(l);
    2145                                if (x <= vl) {
    2146                                        if (x == vl && indexes.getAbs(l) < 0)
    2147                                                indexes.setAbs(l, it.index);
    2148                                        continue;
    2149                                }
    2150                                int h = n - 1;
    2151                                double vh = values.getDouble(h);
    2152                                if (x >= vh) {
    2153                                        if (x == vh && indexes.getAbs(h) < 0)
    2154                                                indexes.setAbs(h, it.index);
    2155                                        continue;
    2156                                }
    2157                                while (h - l > 1) {
    2158                                        int m = (l + h) / 2;
    2159                                        double vm = values.getDouble(m);
    2160                                        if (x < vm) {
    2161                                                h = m;
    2162                                        } else if (x > vm) {
    2163                                                l = m;
    2164                                        } else {
    2165                                                if (indexes.getAbs(m) < 0)
    2166                                                        indexes.setAbs(m, it.index);
    2167                                                break;
    2168                                        }
    2169                                }
    2170                        }
    2171                }
    2172                return indexes;
    2173        }
    2174
    2175        /**
    2176         * Find indexes in sorted dataset of values for each value in other dataset
    2177         * @param a dataset
    2178         * @param values sorted 1D dataset of values to find
    2179         * @return absolute indexes of values (-1 is used to indicate value not found)
    2180         */
    2181        public static IntegerDataset findIndexesForValues(final Dataset a, final Dataset values) {
    2182                if (values.getRank() != 1) {
    2183                        throw new IllegalArgumentException("Values dataset must be 1D");
    2184                }
    2185                IntegerDataset indexes = new IntegerDataset(a.getSize());
    2186                indexes.fill(-1);
    2187
    2188                IndexIterator it = a.getIterator();
    2189                int i = -1;
    2190                final int n = values.getSize();
    2191                if (values instanceof LongDataset) {
    2192                        while (it.hasNext()) {
    2193                                i++;
    2194                                long x = a.getElementLongAbs(it.index);
    2195
    2196                                int l = 0; // binary search to find value in sorted dataset
    2197                                long vl = values.getLong(l);
    2198                                if (x <= vl) {
    2199                                        if (x == vl)
    2200                                                indexes.setAbs(i, l);
    2201                                        continue;
    2202                                }
    2203                                int h = n - 1;
    2204                                long vh = values.getLong(h);
    2205                                if (x >= vh) {
    2206                                        if (x == vh)
    2207                                                indexes.setAbs(i, h);
    2208                                        continue;
    2209                                }
    2210                                while (h - l > 1) {
    2211                                        int m = (l + h) / 2;
    2212                                        long vm = values.getLong(m);
    2213                                        if (x < vm) {
    2214                                                h = m;
    2215                                        } else if (x > vm) {
    2216                                                l = m;
    2217                                        } else {
    2218                                                indexes.setAbs(i, m);
    2219                                                break;
    2220                                        }
    2221                                }
    2222                        }
    2223                } else {
    2224                        while (it.hasNext()) {
    2225                                i++;
    2226                                double x = a.getElementDoubleAbs(it.index);
    2227
    2228                                int l = 0; // binary search to find value in sorted dataset
    2229                                double vl = values.getDouble(l);
    2230                                if (x <= vl) {
    2231                                        if (x == vl)
    2232                                                indexes.setAbs(i, l);
    2233                                        continue;
    2234                                }
    2235                                int h = n - 1;
    2236                                double vh = values.getDouble(h);
    2237                                if (x >= vh) {
    2238                                        if (x == vh)
    2239                                                indexes.setAbs(i, h);
    2240                                        continue;
    2241                                }
    2242                                while (h - l > 1) {
    2243                                        int m = (l + h) / 2;
    2244                                        double vm = values.getDouble(m);
    2245                                        if (x < vm) {
    2246                                                h = m;
    2247                                        } else if (x > vm) {
    2248                                                l = m;
    2249                                        } else {
    2250                                                indexes.setAbs(i, m);
    2251                                                break;
    2252                                        }
    2253                                }
    2254                        }
    2255                }
    2256
    2257                return indexes;
    2258        }
    2259
    2260        /**
    2261         * Roll items over given axis by given amount
    2262         * @param <T> dataset class
    2263         * @param a dataset
    2264         * @param shift amount to shift
    2265         * @param axis if null, then roll flattened dataset
    2266         * @return rolled dataset
    2267         */
    2268        public static <T extends Dataset> T roll(final T a, final int shift, Integer axis) {
    2269                T r = DatasetFactory.zeros(a);
    2270                int is = a.getElementsPerItem();
    2271                if (axis == null) {
    2272                        IndexIterator it = a.getIterator();
    2273                        int s = r.getSize();
    2274                        int i = shift % s;
    2275                        if (i < 0)
    2276                                i += s;
    2277                        while (it.hasNext()) {
    2278                                r.setObjectAbs(i, a.getObjectAbs(it.index));
    2279                                i += is;
    2280                                if (i >= s) {
    2281                                        i %= s;
    2282                                }
    2283                        }
    2284                } else {
    2285                        axis = a.checkAxis(axis);
    2286                        PositionIterator pi = a.getPositionIterator(axis);
    2287                        int s = a.getShapeRef()[axis];
    2288                        Dataset u = DatasetFactory.zeros(is, a.getClass(), new int[] {s});
    2289                        Dataset v = DatasetFactory.zeros(u);
    2290                        int[] pos = pi.getPos();
    2291                        boolean[] hit = pi.getOmit();
    2292                        while (pi.hasNext()) {
    2293                                a.copyItemsFromAxes(pos, hit, u);
    2294                                int i = shift % s;
    2295                                if (i < 0)
    2296                                        i += s;
    2297                                for (int j = 0; j < s; j++) {
    2298                                        v.setObjectAbs(i, u.getObjectAbs(j*is));
    2299                                        i += is;
    2300                                        if (i >= s) {
    2301                                                i %= s;
    2302                                        }
    2303                                }
    2304                                r.setItemsOnAxes(pos, hit, v.getBuffer());
    2305                        }
    2306                }
    2307                return r;
    2308        }
    2309
    2310        /**
    2311         * Roll the specified axis backwards until it lies in given position
    2312         * @param <T> dataset class
    2313         * @param a dataset
    2314         * @param axis The rolled axis (index in shape array). Other axes are left unchanged in relative positions
    2315         * @param start The position with it right of the destination of the rolled axis
    2316         * @return dataset with rolled axis
    2317         */
    2318        @SuppressWarnings("unchecked")
    2319        public static <T extends Dataset> T rollAxis(final T a, int axis, int start) {
    2320                int r = a.getRank();
    2321                axis = a.checkAxis(axis);
    2322                if (start < 0)
    2323                        start += r;
    2324                if (start < 0 || start > r) {
    2325                        throw new IllegalArgumentException("Start is out of range: it should be >= 0 and <= " + r);
    2326                }
    2327                if (axis < start)
    2328                        start--;
    2329
    2330                if (axis == start)
    2331                        return a;
    2332
    2333                ArrayList<Integer> axes = new ArrayList<Integer>();
    2334                for (int i = 0; i < r; i++) {
    2335                        if (i != axis) {
    2336                                axes.add(i);
    2337                        }
    2338                }
    2339                axes.add(start, axis);
    2340                int[] aa = new int[r];
    2341                for (int i = 0; i < r; i++) {
    2342                        aa[i] = axes.get(i);
    2343                }
    2344                return (T) a.getTransposedView(aa);
    2345        }
    2346
    2347        private static SliceND createFlippedSlice(final Dataset a, int axis) {
    2348                int[] shape = a.getShapeRef();
    2349                SliceND slice = new SliceND(shape);
    2350                slice.flip(axis);
    2351                return slice;
    2352        }
    2353
    2354        /**
    2355         * Flip items in left/right direction, column-wise, or along second axis
    2356         * @param <T> dataset class
    2357         * @param a dataset must be at least 2D
    2358         * @return view of flipped dataset
    2359         */
    2360        @SuppressWarnings("unchecked")
    2361        public static <T extends Dataset> T flipLeftRight(final T a) {
    2362                if (a.getRank() < 2) {
    2363                        throw new IllegalArgumentException("Dataset must be at least 2D");
    2364                }
    2365                return (T) a.getSliceView(createFlippedSlice(a, 1));
    2366        }
    2367
    2368        /**
    2369         * Flip items in up/down direction, row-wise, or along first axis
    2370         * @param <T> dataset class
    2371         * @param a dataset
    2372         * @return view of flipped dataset
    2373         */
    2374        @SuppressWarnings("unchecked")
    2375        public static <T extends Dataset> T flipUpDown(final T a) {
    2376                return (T) a.getSliceView(createFlippedSlice(a, 0));
    2377        }
    2378
    2379        /**
    2380         * Rotate items in first two dimension by 90 degrees anti-clockwise
    2381         * @param <T> dataset class
    2382         * @param a dataset must be at least 2D
    2383         * @return view of flipped dataset
    2384         */
    2385        public static <T extends Dataset> T rotate90(final T a) {
    2386                return rotate90(a, 1);
    2387        }
    2388
    2389        /**
    2390         * Rotate items in first two dimension by 90 degrees anti-clockwise
    2391         * @param <T> dataset class
    2392         * @param a dataset must be at least 2D
    2393         * @param k number of 90-degree rotations
    2394         * @return view of flipped dataset
    2395         */
    2396        @SuppressWarnings("unchecked")
    2397        public static <T extends Dataset> T rotate90(final T a, int k) {
    2398                k = k % 4;
    2399                while (k < 0) {
    2400                        k += 4;
    2401                }
    2402                int r = a.getRank();
    2403                if (r < 2) {
    2404                        throw new IllegalArgumentException("Dataset must be at least 2D");
    2405                }
    2406                switch (k) {
    2407                case 1: case 3:
    2408                        int[] axes = new int[r];
    2409                        axes[0] = 1;
    2410                        axes[1] = 0;
    2411                        for (int i = 2; i < r; i++) {
    2412                                axes[i] = i;
    2413                        }
    2414                        Dataset t = a.getTransposedView(axes);
    2415                        return (T) t.getSliceView(createFlippedSlice(t, k == 1 ? 0 : 1));
    2416                case 2:
    2417                        SliceND s = createFlippedSlice(a, 0);
    2418                        s.flip(1);
    2419                        return (T) a.getSliceView(s);
    2420                default:
    2421                case 0:
    2422                        return a;
    2423                }
    2424        }
    2425
    2426        /**
    2427         * Select content according where condition is true. All inputs are broadcasted to a maximum shape
    2428         * @param condition boolean dataset
    2429         * @param x first input
    2430         * @param y second input
    2431         * @return dataset where content is x or y depending on whether condition is true or otherwise
    2432         */
    2433        public static Dataset select(BooleanDataset condition, Object x, Object y) {
    2434                Object[] all = new Object[] {condition, x, y};
    2435                Dataset[] dAll = BroadcastUtils.convertAndBroadcast(all);
    2436                condition = (BooleanDataset) dAll[0];
    2437                Dataset dx = dAll[1];
    2438                Dataset dy = dAll[2];
    2439                Class<? extends Dataset> dc = InterfaceUtils.getBestInterface(dx.getClass(), dy.getClass());
    2440                int ds = Math.max(dx.getElementsPerItem(), dy.getElementsPerItem());
    2441
    2442                Dataset r = DatasetFactory.zeros(ds, dc, condition.getShapeRef());
    2443                IndexIterator iter = condition.getIterator(true);
    2444                final int[] pos = iter.getPos();
    2445                int i = 0;
    2446                while (iter.hasNext()) {
    2447                        r.setObjectAbs(i++, condition.getElementBooleanAbs(iter.index) ? dx.getObject(pos) : dy.getObject(pos));
    2448                }
    2449                return r;
    2450        }
    2451
    2452        /**
    2453         * Select content from choices where condition is true, otherwise use default. All inputs are broadcasted to a maximum shape
    2454         * @param conditions array of boolean datasets
    2455         * @param choices array of datasets or objects
    2456         * @param def default value (can be a dataset)
    2457         * @return dataset
    2458         */
    2459        public static Dataset select(BooleanDataset[] conditions, Object[] choices, Object def) {
    2460                final int n = conditions.length;
    2461                if (choices.length != n) {
    2462                        throw new IllegalArgumentException("Choices list is not same length as conditions list");
    2463                }
    2464                Object[] all = new Object[2*n];
    2465                System.arraycopy(conditions, 0, all, 0, n);
    2466                System.arraycopy(choices, 0, all, n, n);
    2467                Dataset[] dAll = BroadcastUtils.convertAndBroadcast(all);
    2468                conditions = new BooleanDataset[n];
    2469                Dataset[] dChoices = new Dataset[n];
    2470                System.arraycopy(dAll, 0, conditions, 0, n);
    2471                System.arraycopy(dAll, n, dChoices, 0, n);
    2472                Class<? extends Dataset> dc = null;
    2473                int ds = -1;
    2474                for (int i = 0; i < n; i++) {
    2475                        Dataset a = dChoices[i];
    2476                        Class<? extends Dataset> c = a.getClass();
    2477                        dc = InterfaceUtils.getBestInterface(dc, c);
    2478                        int s = a.getElementsPerItem();
    2479                        if (s > ds) {
    2480                                ds = s;
    2481                        }
    2482                }
    2483                if (dc == null || ds < 1) {
    2484                        throw new IllegalArgumentException("Dataset types of choices are invalid");
    2485                }
    2486
    2487                Dataset r = DatasetFactory.zeros(ds, dc, conditions[0].getShapeRef());
    2488                Dataset d = DatasetFactory.createFromObject(def).getBroadcastView(r.getShapeRef());
    2489                PositionIterator iter = new PositionIterator(r.getShapeRef());
    2490                final int[] pos = iter.getPos();
    2491                int i = 0;
    2492                while (iter.hasNext()) {
    2493                        int j = 0;
    2494                        for (; j < n; j++) {
    2495                                if (conditions[j].get(pos)) {
    2496                                        r.setObjectAbs(i++, dChoices[j].getObject(pos));
    2497                                        break;
    2498                                }
    2499                        }
    2500                        if (j == n) {
    2501                                r.setObjectAbs(i++, d.getObject(pos));
    2502                        }
    2503                }
    2504                return r;
    2505        }
    2506
    2507        /**
    2508         * Choose content from choices where condition is true, otherwise use default. All inputs are broadcasted to a maximum shape
    2509         * @param index integer dataset (ideally, items should be in [0, n) range, if there are n choices)
    2510         * @param choices array of datasets or objects
    2511         * @param throwAIOOBE if true, throw array index out of bound exception
    2512         * @param clip true to clip else wrap indices out of bounds; only used when throwAOOBE is false
    2513         * @return dataset
    2514         */
    2515        public static Dataset choose(IntegerDataset index, Object[] choices, boolean throwAIOOBE, boolean clip) {
    2516                final int n = choices.length;
    2517                Object[] all = new Object[n + 1];
    2518                System.arraycopy(choices, 0, all, 0, n);
    2519                all[n] = index;
    2520                Dataset[] dChoices = BroadcastUtils.convertAndBroadcast(all);
    2521                Class<? extends Dataset> dc = null;
    2522                int ds = -1;
    2523                int mr = -1;
    2524                for (int i = 0; i < n; i++) {
    2525                        Dataset a = dChoices[i];
    2526                        int r = a.getRank();
    2527                        if (r > mr) {
    2528                                mr = r;
    2529                        }
    2530                        dc = InterfaceUtils.getBestInterface(dc, a.getClass());
    2531                        int s = a.getElementsPerItem();
    2532                        if (s > ds) {
    2533                                ds = s;
    2534                        }
    2535                }
    2536                if (dc == null || ds < 1) {
    2537                        throw new IllegalArgumentException("Dataset types of choices are invalid");
    2538                }
    2539                index = (IntegerDataset) dChoices[n];
    2540                dChoices[n] = null;
    2541
    2542                Dataset r = DatasetFactory.zeros(ds, dc, index.getShapeRef());
    2543                IndexIterator iter = index.getIterator(true);
    2544                final int[] pos = iter.getPos();
    2545                int i = 0;
    2546                while (iter.hasNext()) {
    2547                        int j = index.getAbs(iter.index);
    2548                        if (j < 0) {
    2549                                if (throwAIOOBE)
    2550                                        throw new ArrayIndexOutOfBoundsException(j);
    2551                                if (clip) {
    2552                                        j = 0;
    2553                                } else {
    2554                                        j %= n;
    2555                                        j += n; // as remainder still negative
    2556                                }
    2557                        }
    2558                        if (j >= n) {
    2559                                if (throwAIOOBE)
    2560                                        throw new ArrayIndexOutOfBoundsException(j);
    2561                                if (clip) {
    2562                                        j = n - 1;
    2563                                } else {
    2564                                        j %= n;
    2565                                }
    2566                        }
    2567                        Dataset c = dChoices[j];
    2568                        r.setObjectAbs(i++, c.getObject(pos));
    2569                }
    2570                return r;
    2571        }
    2572
    2573        /**
    2574         * Calculate positions in given shape from a dataset of 1-D indexes
    2575         * @param indices dataset values taken as integers for index
    2576         * @param shape dataset shape
    2577         * @return list of positions as integer datasets
    2578         */
    2579        public static List<IntegerDataset> calcPositionsFromIndexes(Dataset indices, int[] shape) {
    2580                int rank = shape.length;
    2581                List<IntegerDataset> posns = new ArrayList<IntegerDataset>();
    2582                int[] iShape = indices.getShapeRef();
    2583                for (int i = 0; i < rank; i++) {
    2584                        posns.add(new IntegerDataset(iShape));
    2585                }
    2586                IndexIterator it = indices.getIterator(true);
    2587                int[] pos = it.getPos();
    2588                while (it.hasNext()) {
    2589                        int n = indices.getInt(pos);
    2590                        int[] p = ShapeUtils.getNDPositionFromShape(n, shape);
    2591                        for (int i = 0; i < rank; i++) {
    2592                                posns.get(i).setItem(p[i], pos);
    2593                        }
    2594                }
    2595                return posns;
    2596        }
    2597
    2598
    2599        /**
    2600         * Calculate indexes in given shape from datasets of position
    2601         * @param positions as a list of datasets where each holds the position in a dimension
    2602         * @param shape dataset shape
    2603         * @param mode either null, zero-length, unit length or length of rank of shape where
    2604         *  0 = raise exception, 1 = wrap, 2 = clip
    2605         * @return indexes as an integer dataset
    2606         */
    2607        public static IntegerDataset calcIndexesFromPositions(List<? extends Dataset> positions, int[] shape, int... mode) {
    2608                int rank = shape.length;
    2609                if (positions.size() != rank) {
    2610                        throw new IllegalArgumentException("Number of position datasets must be equal to rank of shape");
    2611                }
    2612
    2613                if (mode == null || mode.length == 0) {
    2614                        mode = new int[rank];
    2615                } else if (mode.length == 1) {
    2616                        int m = mode[0];
    2617                        mode = new int[rank];
    2618                        Arrays.fill(mode, m);
    2619                } else if (mode.length != rank) {
    2620                        throw new IllegalArgumentException("Mode length greater than one must match rank of shape");
    2621                }
    2622                for (int i = 0; i < rank; i++) {
    2623                        int m = mode[i];
    2624                        if (m < 0 || m > 2) {
    2625                                throw new IllegalArgumentException("Unknown mode value - it must be 0, 1, or 2");
    2626                        }
    2627                }
    2628
    2629                Dataset p = positions.get(0);
    2630                IntegerDataset indexes = new IntegerDataset(p.getShapeRef());
    2631                IndexIterator it = p.getIterator(true);
    2632                int[] iPos = it.getPos();
    2633                int[] tPos = new int[rank];
    2634                while (it.hasNext()) {
    2635                        for (int i = 0; i < rank; i++) {
    2636                                p = positions.get(i);
    2637                                int j = p.getInt(iPos);
    2638                                int d = shape[i];
    2639                                if (mode[i] == 0) {
    2640                                        if (j < 0 || j >= d) {
    2641                                                throw new ArrayIndexOutOfBoundsException("Position value exceeds dimension in shape");
    2642                                        }
    2643                                } else if (mode[i] == 1) {
    2644                                        while (j < 0)
    2645                                                j += d;
    2646                                        while (j >= d)
    2647                                                j -= d;
    2648                                } else {
    2649                                        if (j < 0)
    2650                                                j = 0;
    2651                                        if (j >= d)
    2652                                                j = d - 1;
    2653                                }
    2654                                tPos[i] = j;
    2655                        }
    2656                        indexes.set(ShapeUtils.getFlat1DIndex(shape, tPos), iPos);
    2657                }
    2658
    2659                return indexes;
    2660        }
    2661
    2662        /**
    2663         * Serialize dataset by flattening it. Discards metadata
    2664         * @param data dataset
    2665         * @return some java array
    2666         */
    2667        public static Serializable serializeDataset(final IDataset data) {
    2668                Dataset d = convertToDataset(data).getView(false);
    2669                d.clearMetadata(null);
    2670                return d.flatten().getBuffer();
    2671        }
    2672
    2673        /**
    2674         * Extract values where condition is non-zero. This is similar to Dataset#getByBoolean but supports broadcasting
    2675         * @param data dataset
    2676         * @param condition should be broadcastable to data
    2677         * @return 1-D dataset of values
    2678         */
    2679        public static Dataset extract(final IDataset data, final IDataset condition) {
    2680                Dataset a = convertToDataset(data.getSliceView());
    2681                Dataset b = cast(BooleanDataset.class, condition.getSliceView());
    2682
    2683                try {
    2684                        return a.getByBoolean(b);
    2685                } catch (IllegalArgumentException e) {
    2686                        final int length = ((Number) b.sum()).intValue();
    2687
    2688                        BroadcastPairIterator it = new BroadcastPairIterator(a, b, null, false);
    2689                        int size = ShapeUtils.calcSize(it.getShape());
    2690                        Dataset c;
    2691                        if (length < size) {
    2692                                int[] ashape = it.getFirstShape();
    2693                                int[] bshape = it.getSecondShape();
    2694                                int r = ashape.length;
    2695                                size = length;
    2696                                for (int i = 0; i < r; i++) {
    2697                                        int s = ashape[i];
    2698                                        if (s > 1 && bshape[i] == 1) {
    2699                                                size *= s;
    2700                                        }
    2701                                }
    2702                        }
    2703                        c = DatasetFactory.zeros(a.getClass(), size);
    2704
    2705                        int i = 0;
    2706                        if (it.isOutputDouble()) {
    2707                                while (it.hasNext()) {
    2708                                        if (it.bLong != 0) {
    2709                                                c.setObjectAbs(i++, it.aDouble);
    2710                                        }
    2711                                }
    2712                        } else {
    2713                                while (it.hasNext()) {
    2714                                        if (it.bLong != 0) {
    2715                                                c.setObjectAbs(i++, it.aLong);
    2716                                        }
    2717                                }
    2718                        }
    2719
    2720                        return c;
    2721                }
    2722        }
    2723
    2724        /**
    2725         * Set shape to keep original rank
    2726         * @param a dataset
    2727         * @param originalShape original shape
    2728         * @param axes dimensions in original shape to set to 1
    2729         * @since 2.2
    2730         */
    2731        public static void setShapeToOriginalRank(ILazyDataset a, int[] originalShape, int... axes) {
    2732                a.setShape(ShapeUtils.getReducedShapeKeepRank(originalShape, axes));
    2733        }
    2734}