    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.Serializable;
    016import java.lang.annotation.Annotation;
    017import java.lang.reflect.Array;
    018import java.lang.reflect.Field;
    019import java.util.ArrayList;
    020import java.util.Arrays;
    021import java.util.Collection;
    022import java.util.concurrent.ConcurrentMap;
    023import java.util.concurrent.ConcurrentHashMap;
    024import java.util.List;
    025import java.util.Map;
    026
    027import org.eclipse.january.DatasetException;
    028import org.eclipse.january.MetadataException;
    029import org.eclipse.january.metadata.Dirtiable;
    030import org.eclipse.january.metadata.ErrorMetadata;
    031import org.eclipse.january.metadata.IMetadata;
    032import org.eclipse.january.metadata.MetadataFactory;
    033import org.eclipse.january.metadata.MetadataType;
    034import org.eclipse.january.metadata.Reshapeable;
    035import org.eclipse.january.metadata.Sliceable;
    036import org.eclipse.january.metadata.Transposable;
    037import org.slf4j.Logger;
    038import org.slf4j.LoggerFactory;
    039
    040/**
    041 * Common base for both lazy and normal dataset implementations
    042 */
    043public abstract class LazyDatasetBase implements ILazyDataset, Serializable {
    044
    045        private static final long serialVersionUID = 767926846438976050L;
    046
    047        protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class);
    048
    049        protected static boolean catchExceptions;
    050
    051        static {
    052                /**
    053                 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE
    054                 */
    055                try {
    056                        catchExceptions = Boolean.getBoolean("run.in.eclipse");
    057                } catch (SecurityException e) {
    058                        // set a default for when the security manager does not allow access to the requested key
    059                        catchExceptions = false;
    060                }
    061        }
    062
    063        transient private boolean dirty = true; // indicate dirty state of metadata
    064        protected String name = "";
    065
    066        /**
    067         * The shape or dimensions of the dataset
    068         */
    069        protected int[] shape;
    070
    071        protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null;
    072
    073        /**
    074         * @return type of dataset item
    075         */
    076        abstract public int getDType();
    077
    078        @Override
    079        public Class<?> getElementClass() {
    080                return DTypeUtils.getElementClass(getDType());
    081        }
    082
    083        @Override
    084        public LazyDatasetBase clone() {
    085                return null;
    086        }
    087
    088        @Override
    089        public boolean equals(Object obj) {
    090                if (this == obj) {
    091                        return true;
    092                }
    093                if (obj == null) {
    094                        return false;
    095                }
    096                if (!getClass().equals(obj.getClass())) {
    097                        return false;
    098                }
    099
    100                LazyDatasetBase other = (LazyDatasetBase) obj;
    101                if (getDType() != other.getDType()) {
    102                        return false;
    103                }
    104                if (getElementsPerItem() != other.getElementsPerItem()) {
    105                        return false;
    106                }
    107                if (!Arrays.equals(shape, other.shape)) {
    108                        return false;
    109                }
    110                return true;
    111        }
    112
    113        @Override
    114        public int hashCode() {
    115                int hash = getDType() * 17 + getElementsPerItem();
    116                int rank = shape.length;
    117                for (int i = 0; i < rank; i++) {
    118                        hash = hash*17 + shape[i];
    119                }
    120                return hash;
    121        }
    122
    123        @Override
    124        public String getName() {
    125                return name;
    126        }
    127
    128        @Override
    129        public void setName(String name) {
    130                this.name = name;
    131        }
    132
    133        @Override
    134        public int[] getShape() {
    135                return shape.clone();
    136        }
    137
    138        @Override
    139        public int getRank() {
    140                return shape.length;
    141        }
    142
    143        /**
    144         * This method allows anything that dirties the dataset to clear various metadata values
    145         * so that the other methods can work correctly.
    146         * @since 2.1
    147         */
    148        public void setDirty() {
    149                dirty = true;
    150        }
    151
    152        /**
    153         * Find first sub-interface of (or class that directly implements) MetadataType
    154         * @param clazz
    155         * @return sub-interface
    156         * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it
    157         */
    158        @SuppressWarnings("unchecked")
    159        public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) {
    160                if (clazz.equals(MetadataType.class)) {
    161                        throw new IllegalArgumentException("Cannot accept MetadataType");
    162                }
    163
    164                if (clazz.isInterface()) {
    165                        return clazz;
    166                }
    167
    168                if (clazz.isAnonymousClass()) { // special case
    169                        Class<?> s = clazz.getSuperclass();
    170                        if (!s.equals(Object.class)) {
    171                                // only use super class if it is not an anonymous class of an interface
    172                                clazz = (Class<? extends MetadataType>) s;
    173                        }
    174                }
    175
    176                for (Class<?> c : clazz.getInterfaces()) {
    177                        if (c.equals(MetadataType.class)) {
    178                                if (clazz.isAnonymousClass()) {
    179                                        throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType");
    180                                }
    181                                return clazz;
    182                        }
    183                        if (MetadataType.class.isAssignableFrom(c)) {
    184                                return (Class<? extends MetadataType>) c;
    185                        }
    186                }
    187
    188                Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class
    189                if (c != null) {
    190                        return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c);
    191                }
    192
    193                logger.error("Somehow the search for metadata type interface ended in a bad place");
    194                assert false; // should not be able to get here!!!
    195                return null;
    196        }
    197
    198        @Override
    199        public void setMetadata(MetadataType metadata) {
    200                addMetadata(metadata, true);
    201        }
    202
    203        @Override
    204        public void addMetadata(MetadataType metadata) {
    205                addMetadata(metadata, false);
    206        }
    207
    208        private synchronized void addMetadata(MetadataType metadata, boolean clear) {
    209                if (metadata == null) {
    210                        return;
    211                }
    212
    213                if (this.metadata == null) {
    214                        this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    215                }
    216
    217                Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass());
    218                if (!this.metadata.containsKey(clazz)) {
    219                        this.metadata.put(clazz, new ArrayList<MetadataType>());
    220                } else if (clear) {
    221                        this.metadata.get(clazz).clear();
    222                }
    223                this.metadata.get(clazz).add(metadata);
    224
    225                // add for special case of sub-interfaces of IMetadata
    226                if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) {
    227                        clazz = IMetadata.class;
    228                        if (!this.metadata.containsKey(clazz)) {
    229                                this.metadata.put(clazz, new ArrayList<MetadataType>());
    230                        } else if (clear) {
    231                                this.metadata.get(clazz).clear();
    232                        }
    233                        this.metadata.get(clazz).add(metadata);
    234                }
    235        }
    236
    237        @Override
    238        @Deprecated
    239        public synchronized IMetadata getMetadata() {
    240                return getFirstMetadata(IMetadata.class);
    241        }
    242
    243        @SuppressWarnings("unchecked")
    244        @Override
    245        public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException {
    246                if (metadata == null) {
    247                        dirty = false;
    248                        return null;
    249                }
    250
    251                if (dirty) {
    252                        dirtyMetadata();
    253                        dirty = false;
    254                }
    255
    256                if (clazz == null) {
    257                        List<S> all = new ArrayList<S>();
    258                        for (Class<? extends MetadataType> c : metadata.keySet()) {
    259                                all.addAll((Collection<S>) metadata.get(c));
    260                        }
    261                        return all;
    262                }
    263
    264                return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz));
    265        }
    266
    267        @Override
    268        public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) {
    269                try {
    270                        List<S> ml = getMetadata(clazz);
    271                        if (ml == null) {
    272                                return null;
    273                        }
    274                        for (S t : ml) {
    275                                if (clazz.isInstance(t)) {
    276                                        return t;
    277                                }
    278                        }
    279                } catch (Exception e) {
    280                        logger.error("Get metadata failed!",e);
    281                }
    282
    283                return null;
    284        }
    285
    286        @Override
    287        public synchronized void clearMetadata(Class<? extends MetadataType> clazz) {
    288                if (metadata == null) {
    289                        return;
    290                }
    291
    292                if (clazz == null) {
    293                        metadata.clear();
    294                        return;
    295                }
    296
    297                List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz));
    298                if( list != null) {
    299                        list.clear();
    300                }
    301        }
    302
    303        /**
    304         * @since 2.0
    305         */
    306        protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() {
    307                return copyMetadata(metadata);
    308        }
    309
    310        /**
    311         * @since 2.0
    312         */
    313        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) {
    314                if (metadata == null) {
    315                        return null;
    316                }
    317
    318                ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    319                copyMetadata(metadata, map);
    320                return map;
    321        }
    322
    323        private static void copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> inMetadata,
    324                        Map<Class<? extends MetadataType>, List<MetadataType>> outMetadata) {
    325                for (Class<? extends MetadataType> c : inMetadata.keySet()) {
    326                        List<MetadataType> l = inMetadata.get(c);
    327                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
    328                        outMetadata.put(c, nl);
    329                        for (MetadataType m : l) {
    330                                if (m == null || isMetadataDirty(m)) { // skip dirty metadata
    331                                        continue;
    332                                }
    333                                nl.add(m.clone());
    334                        }
    335                }
    336        }
    337
    338        protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) {
    339                copyMetadata(oldMetadata, metadata);
    340        }
    341
    342        /**
    343         * @since 2.2
    344         */
    345        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> getMetadataMap(ILazyDataset a, boolean clone) {
    346                List<MetadataType> all = null;
    347                try {
    348                        all = a.getMetadata(null);
    349                } catch (Exception e) {
    350                }
    351                if (all == null) {
    352                        return null;
    353                }
    354
    355                ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    356
    357                for (MetadataType m : all) {
    358                        if (m == null || isMetadataDirty(m)) { // skip dirty metadata
    359                                continue;
    360                        }
    361                        Class<? extends MetadataType> c = findMetadataTypeSubInterfaces(m.getClass());
    362                        List<MetadataType> l = map.get(c);
    363                        if (l == null) {
    364                                l = new ArrayList<MetadataType>();
    365                                map.put(c, l);
    366                        }
    367                        if (clone) {
    368                                m = m.clone();
    369                        }
    370                        l.add(m);
    371                }
    372                return map;
    373        }
    374
    375        private static boolean isMetadataDirty(MetadataType m) {
    376                Class<? extends MetadataType> c = m.getClass();
    377                for (Field f : c.getDeclaredFields()) {
    378                        if (f.isAnnotationPresent(Dirtiable.class)) {
    379                                Class<?> t = f.getType();
    380                                if (t.equals(boolean.class) || t.equals(Boolean.class)) {
    381                                        try {
    382                                                f.setAccessible(true);
    383                                                Object o = f.get(m);
    384                                                if (o.equals(true)) {
    385                                                        return true;
    386                                                }
    387                                        } catch (Exception e) {
    388                                                logger.debug("Could not retrieve value of dirty variable: {}", c.getCanonicalName(), e);
    389                                        }
    390                                }
    391                        }
    392                }
    393
    394                return false;
    395        }
    396
    397        interface MetadatasetAnnotationOperation {
    398                /**
    399                 * Process value of given field
    400                 * <p>
    401                 * When the field is not a container then the returned value
    402                 * may replace the old value
    403                 * @param f given field
    404                 * @param o value of field
    405                 * @return transformed field
    406                 */
    407                Object processField(Field f, Object o);
    408
    409                /**
    410                 * @return annotated class
    411                 */
    412                Class<? extends Annotation> getAnnClass();
    413
    414                /**
    415                 * @param axis
    416                 * @return number of dimensions to insert or remove
    417                 */
    418                int change(int axis);
    419
    420                /**
    421                 *
    422                 * @return rank or -1 to match
    423                 */
    424                int getNewRank();
    425
    426                /**
    427                 * Run on given lazy dataset
    428                 * @param lz
    429                 * @return
    430                 */
    431                ILazyDataset run(ILazyDataset lz);
    432        }
    433
    434        class MdsSlice implements MetadatasetAnnotationOperation {
    435                private boolean asView;
    436                private SliceND slice;
    437                private int[] oShape;
    438                private long oSize;
    439
    440                public MdsSlice(boolean asView, SliceND slice) {
    441                        this.asView = asView;
    442                        this.slice = slice;
    443                        oShape = slice.getSourceShape();
    444                        oSize = ShapeUtils.calcLongSize(oShape);
    445                }
    446
    447                @Override
    448                public Object processField(Field field, Object o) {
    449                        return o;
    450                }
    451
    452                @Override
    453                public Class<? extends Annotation> getAnnClass() {
    454                        return Sliceable.class;
    455                }
    456
    457                @Override
    458                public int change(int axis) {
    459                        return 0;
    460                }
    461
    462                @Override
    463                public int getNewRank() {
    464                        return -1;
    465                }
    466
    467                @Override
    468                public ILazyDataset run(ILazyDataset lz) {
    469                        int rank = lz.getRank();
    470                        if (slice.getStart().length != rank) {
    471                                throw new IllegalArgumentException("Slice rank does not match dataset!");
    472                        }
    473
    474                        int[] shape = lz.getShape();
    475                        SliceND nslice;
    476                        if (!ShapeUtils.areShapesBroadcastCompatible(oShape, shape)) {
    477                                nslice = new SliceND(shape);
    478                                for (int i = 0; i < rank; i++) {
    479                                        int s = shape[i];
    480                                        int os = oShape[i];
    481                                        if (s >= os) {
    482                                                nslice.setSlice(i, 0, os, 1);
    483                                        } else if (s == 1) {
    484                                                nslice.setSlice(i, 0, 1, 1);
    485                                        } else {
    486                                                throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!");
    487                                        }
    488                                }
    489                                lz = lz.getSliceView(nslice);
    490                        }
    491                        if (lz.getSize() == oSize) {
    492                                nslice = slice;
    493                        } else {
    494                                nslice = slice.clone();
    495                                for (int i = 0; i < rank; i++) {
    496                                        int s = shape[i];
    497                                        if (s >= oShape[i]) {
    498                                                continue;
    499                                        } else if (s == 1) {
    500                                                nslice.setSlice(i, 0, 1, 1);
    501                                        } else {
    502                                                throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!");
    503                                        }
    504                                }
    505                        }
    506
    507                        if (asView || (lz instanceof IDataset)) {
    508                                return lz.getSliceView(nslice);
    509                        }
    510                        try {
    511                                return lz.getSlice(nslice);
    512                        } catch (DatasetException e) {
    513                                logger.error("Could not slice dataset in metadata", e);
    514                                return null;
    515                        }
    516                }
    517        }
    518
    519        class MdsReshape implements MetadatasetAnnotationOperation {
    520                private boolean matchRank;
    521                private int[] oldShape;
    522                private int[] newShape;
    523                boolean onesOnly;
    524                int[] differences;
    525
    526                /*
    527                 * if only ones then record differences (insertions and deletions)
    528                 *
    529                 * if shape changing, find broadcasted dimensions and disallow
    530                 * merging that include those dimensions
    531                 */
    532                public MdsReshape(final int[] oldShape, final int[] newShape) {
    533                        this.oldShape = oldShape;
    534                        this.newShape = newShape;
    535                        differences = null;
    536                }
    537
    538                @Override
    539                public Object processField(Field field, Object o) {
    540                        Annotation a = field.getAnnotation(Reshapeable.class);
    541                        if (a != null) { // cannot be null
    542                                matchRank = ((Reshapeable) a).matchRank();
    543                        }
    544                        return o;
    545                }
    546
    547                @Override
    548                public Class<? extends Annotation> getAnnClass() {
    549                        return Reshapeable.class;
    550                }
    551
    552                @Override
    553                public int change(int axis) {
    554                        if (matchRank) {
    555                                if (differences == null) {
    556                                        init();
    557                                }
    558
    559                                if (onesOnly) {
    560                                        return differences[axis];
    561                                }
    562                                throw new UnsupportedOperationException("TODO support other shape operations");
    563                        }
    564                        return 0;
    565                }
    566
    567                @Override
    568                public int getNewRank() {
    569                        return matchRank ? newShape.length : -1;
    570                }
    571
    572                private void init() {
    573                        int or = oldShape.length - 1;
    574                        int nr = newShape.length - 1;
    575                        if (or < 0 || nr < 0) { // zero-rank shapes
    576                                onesOnly = true;
    577                                differences = new int[1];
    578                                differences[0] = or < 0 ? nr + 1 : or + 1;
    579                                return;
    580                        }
    581                        int ob = 0;
    582                        int nb = 0;
    583                        onesOnly = true;
    584                        do {
    585                                while (oldShape[ob] == 1 && ob < or) {
    586                                        ob++; // next non-unit dimension
    587                                }
    588                                while (newShape[nb] == 1 && nb < nr) {
    589                                        nb++;
    590                                }
    591                                if (oldShape[ob++] != newShape[nb++]) {
    592                                        onesOnly = false;
    593                                        break;
    594                                }
    595                        } while (ob <= or && nb <= nr);
    596
    597                        ob = 0;
    598                        nb = 0;
    599                        differences = new int[or + 2];
    600                        if (onesOnly) {
    601                                // work out unit dimensions removed from or add to old
    602                                int j = 0;
    603                                do {
    604                                        if (oldShape[ob] != 1 && newShape[nb] != 1) {
    605                                                ob++;
    606                                                nb++;
    607                                        } else {
    608                                                while (oldShape[ob] == 1 && ob < or) {
    609                                                        ob++;
    610                                                        differences[j]--;
    611                                                }
    612                                                while (newShape[nb] == 1 && nb < nr) {
    613                                                        nb++;
    614                                                        differences[j]++;
    615                                                }
    616                                        }
    617                                        j++;
    618                                } while (ob <= or && nb <= nr && j <= or);
    619                                while (ob <= or && oldShape[ob] == 1) {
    620                                        ob++;
    621                                        differences[j]--;
    622                                }
    623                                while (nb <= nr && newShape[nb] == 1) {
    624                                        nb++;
    625                                        differences[j]++;
    626                                }
    627                        } else {
    628                                if (matchRank) {
    629                                        logger.error("Combining dimensions is currently not supported");
    630                                        throw new IllegalArgumentException("Combining dimensions is currently not supported");
    631                                }
    632                                // work out mapping: contiguous dimensions can be grouped or split
    633                                while (ob <= or && nb <= nr) {
    634                                        int ol = oldShape[ob];
    635                                        while (ol == 1 && ol <= or) {
    636                                                ob++;
    637                                                ol = oldShape[ob];
    638                                        }
    639                                        int oe = ob + 1;
    640                                        int nl = newShape[nb];
    641                                        while (nl == 1 && nl <= nr) {
    642                                                nb++;
    643                                                nl = newShape[nb];
    644                                        }
    645                                        int ne = nb + 1;
    646                                        if (ol < nl) {
    647                                                differences[ob] = 1;
    648                                                do { // case where new shape combines several dimensions into one dimension
    649                                                        if (oe == (or + 1)) {
    650                                                                break;
    651                                                        }
    652                                                        differences[oe] = 1;
    653                                                        ol *= oldShape[oe++];
    654                                                } while (ol < nl);
    655                                                differences[oe - 1] = oe - ob; // signal end with difference
    656                                                if (nl != ol) {
    657                                                        logger.error("Single dimension is incompatible with subshape");
    658                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
    659                                                }
    660                                        } else if (ol > nl) {
    661                                                do { // case where new shape spreads single dimension over several dimensions
    662                                                        if (ne == (nr + 1)) {
    663                                                                break;
    664                                                        }
    665                                                        nl *= newShape[ne++];
    666                                                } while (nl < ol);
    667                                                if (nl != ol) {
    668                                                        logger.error("Subshape is incompatible with single dimension");
    669                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
    670                                                }
    671
    672                                        }
    673
    674                                        ob = oe;
    675                                        nb = ne;
    676                                }
    677
    678                        }
    679                }
    680
    681                @Override
    682                public ILazyDataset run(ILazyDataset lz) {
    683                        if (differences == null) {
    684                                init();
    685                        }
    686
    687                        int[] lshape = lz.getShape();
    688                        if (Arrays.equals(newShape, lshape)) {
    689                                return lz;
    690                        }
    691                        int or = lz.getRank();
    692                        int nr = newShape.length;
    693                        int[] nshape = new int[nr];
    694                        Arrays.fill(nshape, 1);
    695                        if (onesOnly) {
    696                                // ignore omit removed dimensions
    697                                for (int i = 0, si = 0, di = 0; i < (or+1) && si <= or && di < nr; i++) {
    698                                        int c = differences[i];
    699                                        if (c == 0) {
    700                                                nshape[di++] = lshape[si++];
    701                                        } else if (c > 0) {
    702                                                while (c-- > 0 && di < nr) {
    703                                                        di++;
    704                                                }
    705                                        } else if (c < 0) {
    706                                                si -= c; // remove dimensions by skipping forward in source array
    707                                        }
    708                                }
    709                        } else {
    710                                boolean[] broadcast = new boolean[or];
    711                                for (int ob = 0; ob < or; ob++) {
    712                                        broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1;
    713                                }
    714                                int osize = lz.getSize();
    715
    716                                // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...)
    717                                int ob = 0;
    718                                int nsize = 1;
    719                                for (int i = 0; i < nr; i++) {
    720                                        if (ob < or && broadcast[ob]) {
    721                                                if (differences[ob] != 0) {
    722                                                        logger.error("Metadata contains a broadcast axis which cannot be reshaped");
    723                                                        throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped");
    724                                                }
    725                                        } else {
    726                                                nshape[i] = nsize < osize ? newShape[i] : 1;
    727                                        }
    728                                        nsize *= nshape[i];
    729                                        ob++;
    730                                }
    731                        }
    732
    733                        ILazyDataset nlz = lz.getSliceView();
    734                        if (lz instanceof Dataset) {
    735                                nlz = ((Dataset) lz).reshape(nshape);
    736                        } else {
    737                                nlz = lz.getSliceView();
    738                                nlz.setShape(nshape);
    739                        }
    740                        return nlz;
    741                }
    742        }
    743
    744        class MdsTranspose implements MetadatasetAnnotationOperation {
    745                int[] map;
    746
    747                public MdsTranspose(final int[] axesMap) {
    748                        map = axesMap;
    749                }
    750
    751                @SuppressWarnings({ "rawtypes", "unchecked" })
    752                @Override
    753                public Object processField(Field f, Object o) {
    754                        // reorder arrays and lists according the axes map
    755                        if (o.getClass().isArray()) {
    756                                int l = Array.getLength(o);
    757                                if (l == map.length) {
    758                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
    759                                        for (int i = 0; i < l; i++) {
    760                                                Array.set(narray, i, Array.get(o, map[i]));
    761                                        }
    762                                        for (int i = 0; i < l; i++) {
    763                                                Array.set(o, i, Array.get(narray, i));
    764                                        }
    765                                }
    766                        } else if (o instanceof List<?>) {
    767                                List list = (List) o;
    768                                int l = list.size();
    769                                if (l == map.length) {
    770                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
    771                                        for (int i = 0; i < l; i++) {
    772                                                Array.set(narray, i, list.get(map[i]));
    773                                        }
    774                                        list.clear();
    775                                        for (int i = 0; i < l; i++) {
    776                                                list.add(Array.get(narray, i));
    777                                        }
    778                                }
    779                        }
    780                        return o;
    781                }
    782
    783                @Override
    784                public Class<? extends Annotation> getAnnClass() {
    785                        return Transposable.class;
    786                }
    787
    788                @Override
    789                public int change(int axis) {
    790                        return 0;
    791                }
    792
    793                @Override
    794                public int getNewRank() {
    795                        return -1;
    796                }
    797
    798                @Override
    799                public ILazyDataset run(ILazyDataset lz) {
    800                        return lz.getTransposedView(map);
    801                }
    802        }
    803
    804        class MdsDirty implements MetadatasetAnnotationOperation {
    805
    806                @Override
    807                public Object processField(Field f, Object o) {
    808                        // throw exception if not boolean???
    809                        Class<?> t = f.getType();
    810                        if (t.equals(boolean.class) || t.equals(Boolean.class)) {
    811                                if (o.equals(false)) {
    812                                        o = true;
    813                                }
    814                        }
    815                        return o;
    816                }
    817
    818                @Override
    819                public Class<? extends Annotation> getAnnClass() {
    820                        return Dirtiable.class;
    821                }
    822
    823                @Override
    824                public int change(int axis) {
    825                        return 0;
    826                }
    827
    828                @Override
    829                public int getNewRank() {
    830                        return -1;
    831                }
    832
    833                @Override
    834                public ILazyDataset run(ILazyDataset lz) {
    835                        return lz;
    836                }
    837        }
    838
    839        /**
    840         * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced
    841         * dataset after cloning the metadata
    842         * @param asView if true then just a view
    843         * @param slice
    844         */
    845        protected void sliceMetadata(boolean asView, final SliceND slice) {
    846                processAnnotatedMetadata(new MdsSlice(asView, slice), true);
    847        }
    848
    849        /**
    850         * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing
    851         * or setting the shape
    852         *
    853         * @param newShape
    854         */
    855        protected void reshapeMetadata(final int[] oldShape, final int[] newShape) {
    856                processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true);
    857        }
    858
    859        /**
    860         * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed
    861         * dataset after cloning the metadata
    862         * @param axesMap
    863         */
    864        protected void transposeMetadata(final int[] axesMap) {
    865                processAnnotatedMetadata(new MdsTranspose(axesMap), true);
    866        }
    867
    868        /**
    869         * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified
    870         * @since 2.0
    871         */
    872        protected void dirtyMetadata() {
    873                processAnnotatedMetadata(new MdsDirty(), true);
    874        }
    875
    876        @SuppressWarnings("unchecked")
    877        private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) {
    878                if (metadata == null)
    879                        return;
    880
    881                for (List<MetadataType> l : metadata.values()) {
    882                        for (MetadataType m : l) {
    883                                if (m == null) {
    884                                        continue;
    885                                }
    886
    887                                Class<? extends MetadataType> mc = m.getClass();
    888                                do { // iterate over super-classes
    889                                        processClass(op, m, mc, throwException);
    890                                        Class<?> sclazz = mc.getSuperclass();
    891                                        if (!MetadataType.class.isAssignableFrom(sclazz)) {
    892                                                break;
    893                                        }
    894                                        mc = (Class<? extends MetadataType>) sclazz;
    895                                } while (true);
    896                        }
    897                }
    898        }
    899
    900        @SuppressWarnings({ "unchecked", "rawtypes" })
    901        private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) {
    902                for (Field f : mc.getDeclaredFields()) {
    903                        if (!f.isAnnotationPresent(op.getAnnClass()))
    904                                continue;
    905
    906                        try {
    907                                f.setAccessible(true);
    908                                Object o = f.get(m);
    909                                if (o == null) {
    910                                        continue;
    911                                }
    912
    913                                Object no = op.processField(f, o);
    914                                if (no != o) {
    915                                        f.set(m, no);
    916                                        continue;
    917                                }
    918                                Object r = null;
    919                                if (o instanceof ILazyDataset) {
    920                                        try {
    921                                                f.set(m, op.run((ILazyDataset) o));
    922                                        } catch (Exception e) {
    923                                                logger.error("Problem processing " + o, e);
    924                                                if (!catchExceptions) {
    925                                                        throw e;
    926                                                }
    927                                        }
    928                                } else if (o.getClass().isArray()) {
    929                                        int l = Array.getLength(o);
    930
    931                                        for (int i = 0; r == null && i < l; i++) {
    932                                                r = Array.get(o, i);
    933                                        }
    934                                        int n = op.getNewRank();
    935                                        if (r == null) {
    936                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
    937                                                        f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n));
    938                                                }
    939                                                continue;
    940                                        }
    941                                        if (n < 0) {
    942                                                n = l;
    943                                        }
    944                                        Object narray = Array.newInstance(r.getClass(), n);
    945                                        for (int i = 0, si = 0, di = 0; di < n && si < l; i++) {
    946                                                int c = op.change(i);
    947                                                if (c == 0) {
    948                                                        Array.set(narray, di++, processObject(op, Array.get(o, si++)));
    949                                                } else if (c > 0) {
    950                                                        di += c; // add nulls by skipping forward in destination array
    951                                                } else if (c < 0) {
    952                                                        si -= c; // remove dimensions by skipping forward in source array
    953                                                }
    954                                        }
    955                                        if (n == l) {
    956                                                for (int i = 0; i < l; i++) {
    957                                                        Array.set(o, i, Array.get(narray, i));
    958                                                }
    959                                        } else {
    960                                                f.set(m, narray);
    961                                        }
    962                                } else if (o instanceof List<?>) {
    963                                        List list = (List) o;
    964                                        int l = list.size();
    965
    966                                        for (int i = 0; r == null && i < l; i++) {
    967                                                r = list.get(i);
    968                                        }
    969                                        int n = op.getNewRank();
    970                                        if (r == null) {
    971                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
    972                                                        list.clear();
    973                                                        for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) {
    974                                                                list.add(null);
    975                                                        }
    976                                                }
    977                                                continue;
    978                                        }
    979
    980                                        if (n < 0) {
    981                                                n = l;
    982                                        }
    983                                        Object narray = Array.newInstance(r.getClass(), n);
    984                                        for (int i = 0, si = 0, di = 0; i < l && si < l; i++) {
    985                                                int c = op.change(i);
    986                                                if (c == 0) {
    987                                                        Array.set(narray, di++, processObject(op, list.get(si++)));
    988                                                } else if (c > 0) {
    989                                                        di += c; // add nulls by skipping forward in destination array
    990                                                } else if (c < 0) {
    991                                                        si -= c; // remove dimensions by skipping forward in source array
    992                                                }
    993                                        }
    994                                        list.clear();
    995                                        for (int i = 0; i < n; i++) {
    996                                                list.add(Array.get(narray, i));
    997                                        }
    998                                } else if (o instanceof Map<?,?>) {
    999                                        Map map = (Map) o;
    1000                                        for (Object k : map.keySet()) {
    1001                                                map.put(k, processObject(op, map.get(k)));
    1002                                        }
    1003                                }
    1004                        } catch (Exception e) {
    1005                                logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e);
    1006                                if (throwException) {
    1007                                        throw new RuntimeException(e);
    1008                                }
    1009                        }
    1010                }
    1011        }
    1012
    1013        @SuppressWarnings({ "unchecked", "rawtypes" })
    1014        private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception {
    1015                if (o == null) {
    1016                        return o;
    1017                }
    1018
    1019                if (o instanceof ILazyDataset) {
    1020                        try {
    1021                                return op.run((ILazyDataset) o);
    1022                        } catch (Exception e) {
    1023                                logger.error("Problem processing " + o, e);
    1024                                if (!catchExceptions) {
    1025                                        throw e;
    1026                                }
    1027                        }
    1028                } else if (o.getClass().isArray()) {
    1029                        int l = Array.getLength(o);
    1030                        for (int i = 0; i < l; i++) {
    1031                                Array.set(o, i, processObject(op, Array.get(o, i)));
    1032                        }
    1033                } else if (o instanceof List<?>) {
    1034                        List list = (List) o;
    1035                        for (int i = 0, imax = list.size(); i < imax; i++) {
    1036                                list.set(i, processObject(op, list.get(i)));
    1037                        }
    1038                } else if (o instanceof Map<?,?>) {
    1039                        Map map = (Map) o;
    1040                        for (Object k : map.keySet()) {
    1041                                map.put(k, processObject(op, map.get(k)));
    1042                        }
    1043                }
    1044                return o;
    1045        }
    1046
    1047        protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) {
    1048                ILazyDataset d = null;
    1049                if (blob instanceof ILazyDataset) {
    1050                        d = (ILazyDataset) blob;
    1051                        if (d instanceof IDataset) {
    1052                                Dataset ed = DatasetUtils.convertToDataset((IDataset) d);
    1053                                int is = ed.getElementsPerItem();
    1054                                if (is != 1 && is != getElementsPerItem()) {
    1055                                        throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset");
    1056                                }
    1057                                d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
    1058                        } else if (!keepLazy) {
    1059                                final int is = getElementsPerItem();
    1060                                try {
    1061                                        d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
    1062                                } catch (DatasetException e) {
    1063                                        logger.error("Could not get data from lazy dataset", e);
    1064                                        return null;
    1065                                }
    1066                        }
    1067                } else {
    1068                        final int is = getElementsPerItem();
    1069                        if (is == 1) {
    1070                                d = DatasetFactory.createFromObject(DoubleDataset.class, blob);
    1071                        } else {
    1072                                try {
    1073                                        d = DatasetFactory.createFromObject(is, CompoundDoubleDataset.class, blob);
    1074                                } catch (IllegalArgumentException e) { // if only single value supplied try again
    1075                                        d = DatasetFactory.createFromObject(DoubleDataset.class, blob);
    1076                                }
    1077                        }
    1078                        if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) {
    1079                                d.setShape(shape.clone());
    1080                        }
    1081                }
    1082                List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape());
    1083                d.setShape(s.get(0));
    1084
    1085                return d;
    1086        }
    1087
    1088        @Override
    1089        public void setErrors(Serializable errors) {
    1090                if (shape == null) {
    1091                        throw new IllegalArgumentException("Cannot set errors for null dataset");
    1092                }
    1093                if (errors == null) {
    1094                        clearMetadata(ErrorMetadata.class);
    1095                        return;
    1096                }
    1097                if (errors == this) {
    1098                        logger.warn("Ignoring setting error to itself as this will lead to infinite recursion");
    1099                        return;
    1100                }
    1101
    1102                ILazyDataset errorData = createFromSerializable(errors, true);
    1103
    1104                ErrorMetadata emd = getErrorMetadata();
    1105                if (emd == null) {
    1106                        try {
    1107                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1108                                setMetadata(emd);
    1109                        } catch (MetadataException me) {
    1110                                logger.error("Could not create metadata", me);
    1111                        }
    1112                }
    1113                emd.setError(errorData);
    1114        }
    1115
    1116        protected ErrorMetadata getErrorMetadata() {
    1117                try {
    1118                        List<ErrorMetadata> el = getMetadata(ErrorMetadata.class);
    1119                        if (el != null && !el.isEmpty()) {
    1120                                 return el.get(0);
    1121                        }
    1122                } catch (Exception e) {
    1123                }
    1124                return null;
    1125        }
    1126
    1127        @Override
    1128        public ILazyDataset getErrors() {
    1129                ErrorMetadata emd = getErrorMetadata();
    1130                return emd == null ? null : emd.getError();
    1131        }
    1132
    1133        @Override
    1134        public boolean hasErrors() {
    1135                return LazyDatasetBase.this.getErrors() != null;
    1136        }
    1137
    1138        /**
    1139         * Check permutation axes
    1140         * @param shape
    1141         * @param axes
    1142         * @return cleaned up axes or null if trivial
    1143         */
    1144        public static int[] checkPermutatedAxes(int[] shape, int... axes) {
    1145                int rank = shape == null ? 0 : shape.length;
    1146
    1147                if (axes == null || axes.length == 0) {
    1148                        axes = new int[rank];
    1149                        for (int i = 0; i < rank; i++) {
    1150                                axes[i] = rank - 1 - i;
    1151                        }
    1152                }
    1153
    1154                if (axes.length != rank) {
    1155                        logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank);
    1156                        throw new IllegalArgumentException("axis permutation does not match shape of dataset");
    1157                }
    1158
    1159                // check all permutation values are within bounds
    1160                for (int i = 0; i < rank; i++) {
    1161                        axes[i] = ShapeUtils.checkAxis(rank, axes[i]);
    1162                }
    1163
    1164                // check for a valid permutation (is this an unnecessary restriction?)
    1165                int[] perm = axes.clone();
    1166                Arrays.sort(perm);
    1167
    1168                for (int i = 0; i < rank; i++) {
    1169                        if (perm[i] != i) {
    1170                                logger.error("axis permutation is not valid: it does not contain complete set of axes");
    1171                                throw new IllegalArgumentException("axis permutation does not contain complete set of axes");
    1172                        }
    1173                }
    1174
    1175                if (Arrays.equals(axes, perm)) {
    1176                        return null; // signal identity or trivial permutation
    1177                }
    1178
    1179                return axes;
    1180        }
    1181}