    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.Serializable;
    016import java.lang.annotation.Annotation;
    017import java.lang.reflect.Array;
    018import java.lang.reflect.Field;
    019import java.util.ArrayList;
    020import java.util.Arrays;
    021import java.util.Collection;
    022import java.util.concurrent.ConcurrentMap;
    023import java.util.concurrent.ConcurrentHashMap;
    024import java.util.List;
    025import java.util.Map;
    026
    027import org.eclipse.january.DatasetException;
    028import org.eclipse.january.MetadataException;
    029import org.eclipse.january.metadata.Dirtiable;
    030import org.eclipse.january.metadata.ErrorMetadata;
    031import org.eclipse.january.metadata.IMetadata;
    032import org.eclipse.january.metadata.MetadataFactory;
    033import org.eclipse.january.metadata.MetadataType;
    034import org.eclipse.january.metadata.Reshapeable;
    035import org.eclipse.january.metadata.Sliceable;
    036import org.eclipse.january.metadata.Transposable;
    037import org.slf4j.Logger;
    038import org.slf4j.LoggerFactory;
    039
    040/**
    041 * Common base for both lazy and normal dataset implementations
    042 */
    043public abstract class LazyDatasetBase implements ILazyDataset, Serializable {
    044
    045        private static final long serialVersionUID = 767926846438976050L;
    046
    047        protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class);
    048
    049        protected static boolean catchExceptions;
    050
    051        static {
    052                /**
    053                 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE
    054                 */
    055                try {
    056                        catchExceptions = Boolean.getBoolean("run.in.eclipse");
    057                } catch (SecurityException e) {
    058                        // set a default for when the security manager does not allow access to the requested key
    059                        catchExceptions = false;
    060                }
    061        }
    062
    063        transient private boolean dirty = true; // indicate dirty state of metadata
    064        protected String name = "";
    065
    066        /**
    067         * The shape or dimensions of the dataset
    068         */
    069        protected int[] shape;
    070
    071        protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null;
    072
    073        /**
    074         * @return type of dataset item
    075         */
    076        abstract public int getDType();
    077
    078        @Override
    079        public Class<?> getElementClass() {
    080                return DTypeUtils.getElementClass(getDType());
    081        }
    082
    083        @Override
    084        public LazyDatasetBase clone() {
    085                return null;
    086        }
    087
    088        @Override
    089        public boolean equals(Object obj) {
    090                if (this == obj) {
    091                        return true;
    092                }
    093                if (obj == null) {
    094                        return false;
    095                }
    096                if (!getClass().equals(obj.getClass())) {
    097                        return false;
    098                }
    099
    100                LazyDatasetBase other = (LazyDatasetBase) obj;
    101                if (getDType() != other.getDType()) {
    102                        return false;
    103                }
    104                if (getElementsPerItem() != other.getElementsPerItem()) {
    105                        return false;
    106                }
    107                if (!Arrays.equals(shape, other.shape)) {
    108                        return false;
    109                }
    110                return true;
    111        }
    112
    113        @Override
    114        public int hashCode() {
    115                int hash = getDType() * 17 + getElementsPerItem();
    116                int rank = shape.length;
    117                for (int i = 0; i < rank; i++) {
    118                        hash = hash*17 + shape[i];
    119                }
    120                return hash;
    121        }
    122
    123        @Override
    124        public String getName() {
    125                return name;
    126        }
    127
    128        @Override
    129        public void setName(String name) {
    130                this.name = name;
    131        }
    132
    133        @Override
    134        public int[] getShape() {
    135                return shape.clone();
    136        }
    137
    138        @Override
    139        public int getRank() {
    140                return shape.length;
    141        }
    142
    143        /**
    144         * This method allows anything that dirties the dataset to clear various metadata values
    145         * so that the other methods can work correctly.
    146         * @since 2.1
    147         */
    148        public void setDirty() {
    149                dirty = true;
    150        }
    151
    152        /**
    153         * Find first sub-interface of (or class that directly implements) MetadataType
    154         * @param clazz
    155         * @return sub-interface
    156         * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it
    157         */
    158        @SuppressWarnings("unchecked")
    159        public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) {
    160                if (clazz.equals(MetadataType.class)) {
    161                        throw new IllegalArgumentException("Cannot accept MetadataType");
    162                }
    163
    164                if (clazz.isInterface()) {
    165                        return clazz;
    166                }
    167
    168                if (clazz.isAnonymousClass()) { // special case
    169                        Class<?> s = clazz.getSuperclass();
    170                        if (!s.equals(Object.class)) {
    171                                // only use super class if it is not an anonymous class of an interface
    172                                clazz = (Class<? extends MetadataType>) s;
    173                        }
    174                }
    175
    176                for (Class<?> c : clazz.getInterfaces()) {
    177                        if (c.equals(MetadataType.class)) {
    178                                if (clazz.isAnonymousClass()) {
    179                                        throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType");
    180                                }
    181                                return clazz;
    182                        }
    183                        if (MetadataType.class.isAssignableFrom(c)) {
    184                                return (Class<? extends MetadataType>) c;
    185                        }
    186                }
    187
    188                Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class
    189                if (c != null) {
    190                        return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c);
    191                }
    192
    193                logger.error("Somehow the search for metadata type interface ended in a bad place");
    194                assert false; // should not be able to get here!!!
    195                return null;
    196        }
    197
    198        @Override
    199        public void setMetadata(MetadataType metadata) {
    200                addMetadata(metadata, true);
    201        }
    202
    203        @Override
    204        public void addMetadata(MetadataType metadata) {
    205                addMetadata(metadata, false);
    206        }
    207
    208        private synchronized void addMetadata(MetadataType metadata, boolean clear) {
    209                if (metadata == null)
    210                        return;
    211
    212                if (this.metadata == null) {
    213                        this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    214                }
    215
    216                Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass());
    217                if (!this.metadata.containsKey(clazz)) {
    218                        this.metadata.put(clazz, new ArrayList<MetadataType>());
    219                } else if (clear) {
    220                        this.metadata.get(clazz).clear();
    221                }
    222                this.metadata.get(clazz).add(metadata);
    223
    224                // add for special case of sub-interfaces of IMetadata
    225                if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) {
    226                        clazz = IMetadata.class;
    227                        if (!this.metadata.containsKey(clazz)) {
    228                                this.metadata.put(clazz, new ArrayList<MetadataType>());
    229                        } else if (clear) {
    230                                this.metadata.get(clazz).clear();
    231                        }
    232                        this.metadata.get(clazz).add(metadata);
    233                }
    234        }
    235
    236        @Override
    237        @Deprecated
    238        public synchronized IMetadata getMetadata() {
    239                return getFirstMetadata(IMetadata.class);
    240        }
    241
    242        @SuppressWarnings("unchecked")
    243        @Override
    244        public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException {
    245                if (metadata == null) {
    246                        dirty = false;
    247                        return null;
    248                }
    249
    250                if (dirty) {
    251                        dirtyMetadata();
    252                        dirty = false;
    253                }
    254
    255                if (clazz == null) {
    256                        List<S> all = new ArrayList<S>();
    257                        for (Class<? extends MetadataType> c : metadata.keySet()) {
    258                                all.addAll((Collection<S>) metadata.get(c));
    259                        }
    260                        return all;
    261                }
    262
    263                return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz));
    264        }
    265
    266        @Override
    267        public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) {
    268                try {
    269                        List<S> ml = getMetadata(clazz);
    270                        if (ml == null) {
    271                                return null;
    272                        }
    273                        for (S t : ml) {
    274                                if (clazz.isInstance(t)) {
    275                                        return t;
    276                                }
    277                        }
    278                } catch (Exception e) {
    279                        logger.error("Get metadata failed!",e);
    280                }
    281
    282                return null;
    283        }
    284
    285        @Override
    286        public synchronized void clearMetadata(Class<? extends MetadataType> clazz) {
    287                if (metadata == null)
    288                        return;
    289
    290                if (clazz == null) {
    291                        metadata.clear();
    292                        return;
    293                }
    294
    295                List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz));
    296                if( list != null) {
    297                        list.clear();
    298                }
    299        }
    300
    301        /**
    302         * @since 2.0
    303         */
    304        protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() {
    305                return copyMetadata(metadata, null);
    306        }
    307
    308        /**
    309         * @since 2.0
    310         */
    311        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) {
    312                return copyMetadata(metadata, null);
    313        }
    314
    315        /**
    316         * Copy metadata. If oMetadata is not null, then copy from that when it has the corresponding items
    317         * @since 2.1
    318         * @param metadata
    319         * @param oMetadata can be null
    320         * @return
    321         */
    322        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata, Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata) {
    323                if (metadata == null)
    324                        return null;
    325
    326                ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    327
    328                for (Class<? extends MetadataType> c : metadata.keySet()) {
    329                        List<MetadataType> l = metadata.get(c);
    330                        if (!l.isEmpty() && oMetadata != null && oMetadata.containsKey(c)) { // only override when not empty
    331                                l = oMetadata.get(c);
    332                        }
    333                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
    334                        map.put(c, nl);
    335                        for (MetadataType m : l) {
    336                                nl.add(m.clone());
    337                        }
    338                }
    339                return map;
    340        }
    341
    342        interface MetadatasetAnnotationOperation {
    343                /**
    344                 * Process value of given field
    345                 * <p>
    346                 * When the field is not a container then the returned value
    347                 * may replace the old value
    348                 * @param f given field
    349                 * @param o value of field
    350                 * @return transformed field
    351                 */
    352                Object processField(Field f, Object o);
    353
    354                /**
    355                 * @return annotated class
    356                 */
    357                Class<? extends Annotation> getAnnClass();
    358
    359                /**
    360                 * @param axis
    361                 * @return number of dimensions to insert or remove
    362                 */
    363                int change(int axis);
    364
    365                /**
    366                 *
    367                 * @return rank or -1 to match
    368                 */
    369                int getNewRank();
    370
    371                /**
    372                 * Run on given lazy dataset
    373                 * @param lz
    374                 * @return
    375                 */
    376                ILazyDataset run(ILazyDataset lz);
    377        }
    378
    379        class MdsSlice implements MetadatasetAnnotationOperation {
    380                private boolean asView;
    381                private SliceND slice;
    382                private int[] oShape;
    383                private long oSize;
    384
    385                public MdsSlice(boolean asView, SliceND slice) {
    386                        this.asView = asView;
    387                        this.slice = slice;
    388                        oShape = slice.getSourceShape();
    389                        oSize = ShapeUtils.calcLongSize(oShape);
    390                }
    391
    392                @Override
    393                public Object processField(Field field, Object o) {
    394                        return o;
    395                }
    396
    397                @Override
    398                public Class<? extends Annotation> getAnnClass() {
    399                        return Sliceable.class;
    400                }
    401
    402                @Override
    403                public int change(int axis) {
    404                        return 0;
    405                }
    406
    407                @Override
    408                public int getNewRank() {
    409                        return -1;
    410                }
    411
    412                @Override
    413                public ILazyDataset run(ILazyDataset lz) {
    414                        int rank = lz.getRank();
    415                        if (slice.getStart().length != rank) {
    416                                throw new IllegalArgumentException("Slice rank does not match dataset!");
    417                        }
    418
    419                        int[] shape = lz.getShape();
    420                        SliceND nslice;
    421                        if (!ShapeUtils.areShapesBroadcastCompatible(oShape, shape)) {
    422                                nslice = new SliceND(shape);
    423                                for (int i = 0; i < rank; i++) {
    424                                        int s = shape[i];
    425                                        int os = oShape[i];
    426                                        if (s >= os) {
    427                                                nslice.setSlice(i, 0, os, 1);
    428                                        } else if (s == 1) {
    429                                                nslice.setSlice(i, 0, 1, 1);
    430                                        } else {
    431                                                throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!");
    432                                        }
    433                                }
    434                                lz = lz.getSliceView(nslice);
    435                        }
    436                        if (lz.getSize() == oSize) {
    437                                nslice = slice;
    438                        } else {
    439                                nslice = slice.clone();
    440                        }
    441                        for (int i = 0; i < rank; i++) {
    442                                int s = shape[i];
    443                                if (s >= oShape[i]) {
    444                                        continue;
    445                                } else if (s == 1) {
    446                                        nslice.setSlice(i, 0, 1, 1);
    447                                } else {
    448                                        throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!");
    449                                }
    450                        }
    451
    452                        if (asView || (lz instanceof IDataset)) {
    453                                return lz.getSliceView(nslice);
    454                        }
    455                        try {
    456                                return lz.getSlice(nslice);
    457                        } catch (DatasetException e) {
    458                                logger.error("Could not slice dataset in metadata", e);
    459                                return null;
    460                        }
    461                }
    462        }
    463
    464        class MdsReshape implements MetadatasetAnnotationOperation {
    465                private boolean matchRank;
    466                private int[] oldShape;
    467                private int[] newShape;
    468                boolean onesOnly;
    469                int[] differences;
    470
    471                /*
    472                 * if only ones then record differences (insertions and deletions)
    473                 *
    474                 * if shape changing, find broadcasted dimensions and disallow
    475                 * merging that include those dimensions
    476                 */
    477                public MdsReshape(final int[] oldShape, final int[] newShape) {
    478                        this.oldShape = oldShape;
    479                        this.newShape = newShape;
    480                        differences = null;
    481                }
    482
    483                @Override
    484                public Object processField(Field field, Object o) {
    485                        Annotation a = field.getAnnotation(Reshapeable.class);
    486                        if (a != null) { // cannot be null
    487                                matchRank = ((Reshapeable) a).matchRank();
    488                        }
    489                        return o;
    490                }
    491
    492                @Override
    493                public Class<? extends Annotation> getAnnClass() {
    494                        return Reshapeable.class;
    495                }
    496
    497                @Override
    498                public int change(int axis) {
    499                        if (matchRank) {
    500                                if (differences == null)
    501                                        init();
    502
    503                                if (onesOnly) {
    504                                        return differences[axis];
    505                                }
    506                                throw new UnsupportedOperationException("TODO support other shape operations");
    507                        }
    508                        return 0;
    509                }
    510
    511                @Override
    512                public int getNewRank() {
    513                        return matchRank ? newShape.length : -1;
    514                }
    515
    516                private void init() {
    517                        int or = oldShape.length - 1;
    518                        int nr = newShape.length - 1;
    519                        if (or < 0 || nr < 0) { // zero-rank shapes
    520                                onesOnly = true;
    521                                differences = new int[1];
    522                                differences[0] = or < 0 ? nr + 1 : or + 1;
    523                                return;
    524                        }
    525                        int ob = 0;
    526                        int nb = 0;
    527                        onesOnly = true;
    528                        do {
    529                                while (oldShape[ob] == 1 && ob < or) {
    530                                        ob++; // next non-unit dimension
    531                                }
    532                                while (newShape[nb] == 1 && nb < nr) {
    533                                        nb++;
    534                                }
    535                                if (oldShape[ob++] != newShape[nb++]) {
    536                                        onesOnly = false;
    537                                        break;
    538                                }
    539                        } while (ob <= or && nb <= nr);
    540
    541                        ob = 0;
    542                        nb = 0;
    543                        differences = new int[or + 2];
    544                        if (onesOnly) {
    545                                // work out unit dimensions removed from or add to old
    546                                int j = 0;
    547                                do {
    548                                        if (oldShape[ob] != 1 && newShape[nb] != 1) {
    549                                                ob++;
    550                                                nb++;
    551                                        } else {
    552                                                while (oldShape[ob] == 1 && ob < or) {
    553                                                        ob++;
    554                                                        differences[j]--;
    555                                                }
    556                                                while (newShape[nb] == 1 && nb < nr) {
    557                                                        nb++;
    558                                                        differences[j]++;
    559                                                }
    560                                        }
    561                                        j++;
    562                                } while (ob <= or && nb <= nr && j <= or);
    563                                while (ob <= or && oldShape[ob] == 1) {
    564                                        ob++;
    565                                        differences[j]--;
    566                                }
    567                                while (nb <= nr && newShape[nb] == 1) {
    568                                        nb++;
    569                                        differences[j]++;
    570                                }
    571                        } else {
    572                                if (matchRank) {
    573                                        logger.error("Combining dimensions is currently not supported");
    574                                        throw new IllegalArgumentException("Combining dimensions is currently not supported");
    575                                }
    576                                // work out mapping: contiguous dimensions can be grouped or split
    577                                while (ob <= or && nb <= nr) {
    578                                        int ol = oldShape[ob];
    579                                        while (ol == 1 && ol <= or) {
    580                                                ob++;
    581                                                ol = oldShape[ob];
    582                                        }
    583                                        int oe = ob + 1;
    584                                        int nl = newShape[nb];
    585                                        while (nl == 1 && nl <= nr) {
    586                                                nb++;
    587                                                nl = newShape[nb];
    588                                        }
    589                                        int ne = nb + 1;
    590                                        if (ol < nl) {
    591                                                differences[ob] = 1;
    592                                                do { // case where new shape combines several dimensions into one dimension
    593                                                        if (oe == (or + 1)) {
    594                                                                break;
    595                                                        }
    596                                                        differences[oe] = 1;
    597                                                        ol *= oldShape[oe++];
    598                                                } while (ol < nl);
    599                                                differences[oe - 1] = oe - ob; // signal end with difference
    600                                                if (nl != ol) {
    601                                                        logger.error("Single dimension is incompatible with subshape");
    602                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
    603                                                }
    604                                        } else if (ol > nl) {
    605                                                do { // case where new shape spreads single dimension over several dimensions
    606                                                        if (ne == (nr + 1)) {
    607                                                                break;
    608                                                        }
    609                                                        nl *= newShape[ne++];
    610                                                } while (nl < ol);
    611                                                if (nl != ol) {
    612                                                        logger.error("Subshape is incompatible with single dimension");
    613                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
    614                                                }
    615
    616                                        }
    617
    618                                        ob = oe;
    619                                        nb = ne;
    620                                }
    621
    622                        }
    623                }
    624
    625                @Override
    626                public ILazyDataset run(ILazyDataset lz) {
    627                        if (differences == null)
    628                                init();
    629
    630                        int[] lshape = lz.getShape();
    631                        if (Arrays.equals(newShape, lshape)) {
    632                                return lz;
    633                        }
    634                        int or = lz.getRank();
    635                        int nr = newShape.length;
    636                        int[] nshape = new int[nr];
    637                        Arrays.fill(nshape, 1);
    638                        if (onesOnly) {
    639                                // ignore omit removed dimensions
    640                                for (int i = 0, si = 0, di = 0; i < (or+1) && si <= or && di < nr; i++) {
    641                                        int c = differences[i];
    642                                        if (c == 0) {
    643                                                nshape[di++] = lshape[si++];
    644                                        } else if (c > 0) {
    645                                                while (c-- > 0 && di < nr) {
    646                                                        di++;
    647                                                }
    648                                        } else if (c < 0) {
    649                                                si -= c; // remove dimensions by skipping forward in source array
    650                                        }
    651                                }
    652                        } else {
    653                                boolean[] broadcast = new boolean[or];
    654                                for (int ob = 0; ob < or; ob++) {
    655                                        broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1;
    656                                }
    657                                int osize = lz.getSize();
    658
    659                                // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...)
    660                                int ob = 0;
    661                                int nsize = 1;
    662                                for (int i = 0; i < nr; i++) {
    663                                        if (ob < or && broadcast[ob]) {
    664                                                if (differences[ob] != 0) {
    665                                                        logger.error("Metadata contains a broadcast axis which cannot be reshaped");
    666                                                        throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped");
    667                                                }
    668                                        } else {
    669                                                nshape[i] = nsize < osize ? newShape[i] : 1;
    670                                        }
    671                                        nsize *= nshape[i];
    672                                        ob++;
    673                                }
    674                        }
    675
    676                        ILazyDataset nlz = lz.getSliceView();
    677                        if (lz instanceof Dataset) {
    678                                nlz = ((Dataset) lz).reshape(nshape);
    679                        } else {
    680                                nlz = lz.getSliceView();
    681                                nlz.setShape(nshape);
    682                        }
    683                        return nlz;
    684                }
    685        }
    686
    687        class MdsTranspose implements MetadatasetAnnotationOperation {
    688                int[] map;
    689
    690                public MdsTranspose(final int[] axesMap) {
    691                        map = axesMap;
    692                }
    693
    694                @SuppressWarnings({ "rawtypes", "unchecked" })
    695                @Override
    696                public Object processField(Field f, Object o) {
    697                        // reorder arrays and lists according the axes map
    698                        if (o.getClass().isArray()) {
    699                                int l = Array.getLength(o);
    700                                if (l == map.length) {
    701                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
    702                                        for (int i = 0; i < l; i++) {
    703                                                Array.set(narray, i, Array.get(o, map[i]));
    704                                        }
    705                                        for (int i = 0; i < l; i++) {
    706                                                Array.set(o, i, Array.get(narray, i));
    707                                        }
    708                                }
    709                        } else if (o instanceof List<?>) {
    710                                List list = (List) o;
    711                                int l = list.size();
    712                                if (l == map.length) {
    713                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
    714                                        for (int i = 0; i < l; i++) {
    715                                                Array.set(narray, i, list.get(map[i]));
    716                                        }
    717                                        list.clear();
    718                                        for (int i = 0; i < l; i++) {
    719                                                list.add(Array.get(narray, i));
    720                                        }
    721                                }
    722                        }
    723                        return o;
    724                }
    725
    726                @Override
    727                public Class<? extends Annotation> getAnnClass() {
    728                        return Transposable.class;
    729                }
    730
    731                @Override
    732                public int change(int axis) {
    733                        return 0;
    734                }
    735
    736                @Override
    737                public int getNewRank() {
    738                        return -1;
    739                }
    740
    741                @Override
    742                public ILazyDataset run(ILazyDataset lz) {
    743                        return lz.getTransposedView(map);
    744                }
    745        }
    746
    747        class MdsDirty implements MetadatasetAnnotationOperation {
    748
    749                @Override
    750                public Object processField(Field f, Object o) {
    751                        // throw exception if not boolean???
    752                        Class<?> t = f.getType();
    753                        if (t.equals(boolean.class) || t.equals(Boolean.class)) {
    754                                if (o.equals(false)) {
    755                                        o = true;
    756                                }
    757                        }
    758                        return o;
    759                }
    760
    761                @Override
    762                public Class<? extends Annotation> getAnnClass() {
    763                        return Dirtiable.class;
    764                }
    765
    766                @Override
    767                public int change(int axis) {
    768                        return 0;
    769                }
    770
    771                @Override
    772                public int getNewRank() {
    773                        return -1;
    774                }
    775
    776                @Override
    777                public ILazyDataset run(ILazyDataset lz) {
    778                        return lz;
    779                }
    780        }
    781
    782        /**
    783         * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced
    784         * dataset after cloning the metadata
    785         * @param asView if true then just a view
    786         * @param slice
    787         */
    788        protected void sliceMetadata(boolean asView, final SliceND slice) {
    789                processAnnotatedMetadata(new MdsSlice(asView, slice), true);
    790        }
    791
    792        /**
    793         * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing
    794         * or setting the shape
    795         *
    796         * @param newShape
    797         */
    798        protected void reshapeMetadata(final int[] oldShape, final int[] newShape) {
    799                processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true);
    800        }
    801
    802        /**
    803         * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed
    804         * dataset after cloning the metadata
    805         * @param axesMap
    806         */
    807        protected void transposeMetadata(final int[] axesMap) {
    808                processAnnotatedMetadata(new MdsTranspose(axesMap), true);
    809        }
    810
    811        /**
    812         * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified
    813         * @since 2.0
    814         */
    815        protected void dirtyMetadata() {
    816                processAnnotatedMetadata(new MdsDirty(), true);
    817        }
    818
    819        @SuppressWarnings("unchecked")
    820        private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) {
    821                if (metadata == null)
    822                        return;
    823
    824                for (Class<? extends MetadataType> c : metadata.keySet()) {
    825                        for (MetadataType m : metadata.get(c)) {
    826                                if (m == null)
    827                                        continue;
    828
    829                                Class<? extends MetadataType> mc = m.getClass();
    830                                do { // iterate over super-classes
    831                                        processClass(op, m, mc, throwException);
    832                                        Class<?> sclazz = mc.getSuperclass();
    833                                        if (!MetadataType.class.isAssignableFrom(sclazz))
    834                                                break;
    835                                        mc = (Class<? extends MetadataType>) sclazz;
    836                                } while (true);
    837                        }
    838                }
    839        }
    840
    841        @SuppressWarnings({ "unchecked", "rawtypes" })
    842        private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) {
    843                for (Field f : mc.getDeclaredFields()) {
    844                        if (!f.isAnnotationPresent(op.getAnnClass()))
    845                                continue;
    846
    847                        try {
    848                                f.setAccessible(true);
    849                                Object o = f.get(m);
    850                                if (o == null)
    851                                        continue;
    852
    853                                Object no = op.processField(f, o);
    854                                if (no != o) {
    855                                        f.set(m, no);
    856                                        continue;
    857                                }
    858                                Object r = null;
    859                                if (o instanceof ILazyDataset) {
    860                                        try {
    861                                                f.set(m, op.run((ILazyDataset) o));
    862                                        } catch (Exception e) {
    863                                                logger.error("Problem processing " + o, e);
    864                                                if (!catchExceptions)
    865                                                        throw e;
    866                                        }
    867                                } else if (o.getClass().isArray()) {
    868                                        int l = Array.getLength(o);
    869                                        if (l <= 0)
    870                                                continue;
    871
    872                                        for (int i = 0; r == null && i < l; i++) {
    873                                                r = Array.get(o, i);
    874                                        }
    875                                        int n = op.getNewRank();
    876                                        if (r == null) {
    877                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
    878                                                        f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n));
    879                                                }
    880                                                continue;
    881                                        }
    882                                        if (n < 0)
    883                                                n = l;
    884                                        Object narray = Array.newInstance(r.getClass(), n);
    885                                        for (int i = 0, si = 0, di = 0; di < n && si < l; i++) {
    886                                                int c = op.change(i);
    887                                                if (c == 0) {
    888                                                        Array.set(narray, di++, processObject(op, Array.get(o, si++)));
    889                                                } else if (c > 0) {
    890                                                        di += c; // add nulls by skipping forward in destination array
    891                                                } else if (c < 0) {
    892                                                        si -= c; // remove dimensions by skipping forward in source array
    893                                                }
    894                                        }
    895                                        if (n == l) {
    896                                                for (int i = 0; i < l; i++) {
    897                                                        Array.set(o, i, Array.get(narray, i));
    898                                                }
    899                                        } else {
    900                                                f.set(m, narray);
    901                                        }
    902                                } else if (o instanceof List<?>) {
    903                                        List list = (List) o;
    904                                        int l = list.size();
    905                                        if (l <= 0)
    906                                                continue;
    907
    908                                        for (int i = 0; r == null && i < l; i++) {
    909                                                r = list.get(i);
    910                                        }
    911                                        int n = op.getNewRank();
    912                                        if (r == null) {
    913                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
    914                                                        list.clear();
    915                                                        for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) {
    916                                                                list.add(null);
    917                                                        }
    918                                                }
    919                                                continue;
    920                                        }
    921
    922                                        if (n < 0)
    923                                                n = l;
    924                                        Object narray = Array.newInstance(r.getClass(), n);
    925                                        for (int i = 0, si = 0, di = 0; i < l && si < l; i++) {
    926                                                int c = op.change(i);
    927                                                if (c == 0) {
    928                                                        Array.set(narray, di++, processObject(op, list.get(si++)));
    929                                                } else if (c > 0) {
    930                                                        di += c; // add nulls by skipping forward in destination array
    931                                                } else if (c < 0) {
    932                                                        si -= c; // remove dimensions by skipping forward in source array
    933                                                }
    934                                        }
    935                                        list.clear();
    936                                        for (int i = 0; i < n; i++) {
    937                                                list.add(Array.get(narray, i));
    938                                        }
    939                                } else if (o instanceof Map<?,?>) {
    940                                        Map map = (Map) o;
    941                                        for (Object k : map.keySet()) {
    942                                                map.put(k, processObject(op, map.get(k)));
    943                                        }
    944                                }
    945                        } catch (Exception e) {
    946                                logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e);
    947                                if (throwException)
    948                                        throw new RuntimeException(e);
    949                        }
    950                }
    951        }
    952
    953        @SuppressWarnings({ "unchecked", "rawtypes" })
    954        private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception {
    955                if (o == null)
    956                        return o;
    957
    958                if (o instanceof ILazyDataset) {
    959                        try {
    960                                return op.run((ILazyDataset) o);
    961                        } catch (Exception e) {
    962                                logger.error("Problem processing " + o, e);
    963                                if (!catchExceptions)
    964                                        throw e;
    965                        }
    966                } else if (o.getClass().isArray()) {
    967                        int l = Array.getLength(o);
    968                        for (int i = 0; i < l; i++) {
    969                                Array.set(o, i, processObject(op, Array.get(o, i)));
    970                        }
    971                } else if (o instanceof List<?>) {
    972                        List list = (List) o;
    973                        for (int i = 0, imax = list.size(); i < imax; i++) {
    974                                list.set(i, processObject(op, list.get(i)));
    975                        }
    976                } else if (o instanceof Map<?,?>) {
    977                        Map map = (Map) o;
    978                        for (Object k : map.keySet()) {
    979                                map.put(k, processObject(op, map.get(k)));
    980                        }
    981                }
    982                return o;
    983        }
    984
    985        protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) {
    986                for (Class<? extends MetadataType> mc : oldMetadata.keySet()) {
    987                        metadata.put(mc, oldMetadata.get(mc));
    988                }
    989        }
    990
    991        protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) {
    992                ILazyDataset d = null;
    993                if (blob instanceof ILazyDataset) {
    994                        d = (ILazyDataset) blob;
    995                        if (d instanceof IDataset) {
    996                                Dataset ed = DatasetUtils.convertToDataset((IDataset) d);
    997                                int is = ed.getElementsPerItem();
    998                                if (is != 1 && is != getElementsPerItem()) {
    999                                        throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset");
    1000                                }
    1001                                d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
    1002                        } else if (!keepLazy) {
    1003                                final int is = getElementsPerItem();
    1004                                try {
    1005                                        d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
    1006                                } catch (DatasetException e) {
    1007                                        logger.error("Could not get data from lazy dataset", e);
    1008                                        return null;
    1009                                }
    1010                        }
    1011                } else {
    1012                        final int is = getElementsPerItem();
    1013                        if (is == 1) {
    1014                                d = DatasetFactory.createFromObject(DoubleDataset.class, blob);
    1015                        } else {
    1016                                try {
    1017                                        d = DatasetFactory.createFromObject(is, CompoundDoubleDataset.class, blob);
    1018                                } catch (IllegalArgumentException e) { // if only single value supplied try again
    1019                                        d = DatasetFactory.createFromObject(DoubleDataset.class, blob);
    1020                                }
    1021                        }
    1022                        if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) {
    1023                                d.setShape(shape.clone());
    1024                        }
    1025                }
    1026                List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape());
    1027                d.setShape(s.get(0));
    1028
    1029                return d;
    1030        }
    1031
    1032        @Override
    1033        public void setErrors(Serializable errors) {
    1034                if (shape == null) {
    1035                        throw new IllegalArgumentException("Cannot set errors for null dataset");
    1036                }
    1037                if (errors == null) {
    1038                        clearMetadata(ErrorMetadata.class);
    1039                        return;
    1040                }
    1041                if (errors == this) {
    1042                        logger.warn("Ignoring setting error to itself as this will lead to infinite recursion");
    1043                        return;
    1044                }
    1045
    1046                ILazyDataset errorData = createFromSerializable(errors, true);
    1047
    1048                ErrorMetadata emd = getErrorMetadata();
    1049                if (emd == null) {
    1050                        try {
    1051                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1052                                setMetadata(emd);
    1053                        } catch (MetadataException me) {
    1054                                logger.error("Could not create metadata", me);
    1055                        }
    1056                }
    1057                emd.setError(errorData);
    1058        }
    1059
    1060        protected ErrorMetadata getErrorMetadata() {
    1061                try {
    1062                        List<ErrorMetadata> el = getMetadata(ErrorMetadata.class);
    1063                        if (el != null && !el.isEmpty()) {
    1064                                 return el.get(0);
    1065                        }
    1066                } catch (Exception e) {
    1067                }
    1068                return null;
    1069        }
    1070
    1071        @Override
    1072        public ILazyDataset getErrors() {
    1073                ErrorMetadata emd = getErrorMetadata();
    1074                return emd == null ? null : emd.getError();
    1075        }
    1076
    1077        @Override
    1078        public boolean hasErrors() {
    1079                return LazyDatasetBase.this.getErrors() != null;
    1080        }
    1081
    1082        /**
    1083         * Check permutation axes
    1084         * @param shape
    1085         * @param axes
    1086         * @return cleaned up axes or null if trivial
    1087         */
    1088        public static int[] checkPermutatedAxes(int[] shape, int... axes) {
    1089                int rank = shape == null ? 0 : shape.length;
    1090
    1091                if (axes == null || axes.length == 0) {
    1092                        axes = new int[rank];
    1093                        for (int i = 0; i < rank; i++) {
    1094                                axes[i] = rank - 1 - i;
    1095                        }
    1096                }
    1097
    1098                if (axes.length != rank) {
    1099                        logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank);
    1100                        throw new IllegalArgumentException("axis permutation does not match shape of dataset");
    1101                }
    1102
    1103                // check all permutation values are within bounds
    1104                for (int i = 0; i < rank; i++) {
    1105                        axes[i] = ShapeUtils.checkAxis(rank, axes[i]);
    1106                }
    1107
    1108                // check for a valid permutation (is this an unnecessary restriction?)
    1109                int[] perm = axes.clone();
    1110                Arrays.sort(perm);
    1111
    1112                for (int i = 0; i < rank; i++) {
    1113                        if (perm[i] != i) {
    1114                                logger.error("axis permutation is not valid: it does not contain complete set of axes");
    1115                                throw new IllegalArgumentException("axis permutation does not contain complete set of axes");
    1116                        }
    1117                }
    1118
    1119                if (Arrays.equals(axes, perm))
    1120                        return null; // signal identity or trivial permutation
    1121
    1122                return axes;
    1123        }
    1124}