    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.Serializable;
    016import java.lang.annotation.Annotation;
    017import java.lang.reflect.Array;
    018import java.lang.reflect.Field;
    019import java.util.ArrayList;
    020import java.util.Arrays;
    021import java.util.Collection;
    022import java.util.List;
    023import java.util.Map;
    024import java.util.concurrent.ConcurrentHashMap;
    025import java.util.concurrent.ConcurrentMap;
    026
    027import org.eclipse.january.DatasetException;
    028import org.eclipse.january.MetadataException;
    029import org.eclipse.january.metadata.Dirtiable;
    030import org.eclipse.january.metadata.ErrorMetadata;
    031import org.eclipse.january.metadata.IMetadata;
    032import org.eclipse.january.metadata.MetadataFactory;
    033import org.eclipse.january.metadata.MetadataType;
    034import org.eclipse.january.metadata.Reshapeable;
    035import org.eclipse.january.metadata.Sliceable;
    036import org.eclipse.january.metadata.Transposable;
    037import org.slf4j.Logger;
    038import org.slf4j.LoggerFactory;
    039
    040/**
    041 * Common base for both lazy and normal dataset implementations
    042 */
    043public abstract class LazyDatasetBase implements ILazyDataset, Serializable {
    044
    045        private static final long serialVersionUID = 767926846438976050L;
    046
    047        private static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class);
    048
    049        transient private boolean dirty = true; // indicate dirty state of metadata
    050        protected String name = "";
    051
    052        /**
    053         * The shape or dimensions of the dataset
    054         */
    055        protected int[] shape;
    056
    057        protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null;
    058
    059        /**
    060         * @return type of dataset item
    061         */
    062        abstract public int getDType();
    063
    064        @Override
    065        public LazyDatasetBase clone() {
    066                return null;
    067        }
    068
    069        @Override
    070        public boolean equals(Object obj) {
    071                if (this == obj) {
    072                        return true;
    073                }
    074                if (obj == null) {
    075                        return false;
    076                }
    077                if (!getClass().equals(obj.getClass())) {
    078                        return false;
    079                }
    080
    081                LazyDatasetBase other = (LazyDatasetBase) obj;
    082                if (getDType() != other.getDType()) {
    083                        return false;
    084                }
    085                if (getElementsPerItem() != other.getElementsPerItem()) {
    086                        return false;
    087                }
    088                if (!Arrays.equals(shape, other.shape)) {
    089                        return false;
    090                }
    091                return true;
    092        }
    093
    094        @Override
    095        public int hashCode() {
    096                int hash = getDType() * 17 + getElementsPerItem();
    097                int rank = shape.length;
    098                for (int i = 0; i < rank; i++) {
    099                        hash = hash*17 + shape[i];
    100                }
    101                return hash;
    102        }
    103
    104        @Override
    105        public String getName() {
    106                return name;
    107        }
    108
    109        @Override
    110        public void setName(String name) {
    111                this.name = name;
    112        }
    113
    114        @Override
    115        public int[] getShape() {
    116                return shape.clone();
    117        }
    118
    119        @Override
    120        public int getRank() {
    121                return shape.length;
    122        }
    123
    124        /**
    125         * This method allows anything that dirties the dataset to clear various metadata values
    126         * so that the other methods can work correctly.
    127         * @since 2.1
    128         */
    129        public void setDirty() {
    130                dirty = true;
    131        }
    132
    133        /**
    134         * Check if slice is compatible with dataset's shape
    135         * @param slice to check
    136         */
    137        protected void checkSliceND(SliceND slice) {
    138                slice.checkShapes(shape, null);
    139        }
    140
    141        /**
    142         * Find first sub-interface of (or class that directly implements) MetadataType
    143         * @param clazz metadata type
    144         * @return sub-interface
    145         * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it
    146         */
    147        @SuppressWarnings("unchecked")
    148        public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) {
    149                if (clazz.equals(MetadataType.class)) {
    150                        throw new IllegalArgumentException("Cannot accept MetadataType");
    151                }
    152
    153                if (clazz.isInterface()) {
    154                        return clazz;
    155                }
    156
    157                if (clazz.isAnonymousClass()) { // special case
    158                        Class<?> s = clazz.getSuperclass();
    159                        if (!s.equals(Object.class)) {
    160                                // only use super class if it is not an anonymous class of an interface
    161                                clazz = (Class<? extends MetadataType>) s;
    162                        }
    163                }
    164
    165                for (Class<?> c : clazz.getInterfaces()) {
    166                        if (c.equals(MetadataType.class)) {
    167                                if (clazz.isAnonymousClass()) {
    168                                        throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType");
    169                                }
    170                                return clazz;
    171                        }
    172                        if (MetadataType.class.isAssignableFrom(c)) {
    173                                return (Class<? extends MetadataType>) c;
    174                        }
    175                }
    176
    177                Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class
    178                if (c != null) {
    179                        return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c);
    180                }
    181
    182                logger.error("Somehow the search for metadata type interface ended in a bad place");
    183                assert false; // should not be able to get here!!!
    184                return null;
    185        }
    186
    187        @Override
    188        public void setMetadata(MetadataType metadata) {
    189                addMetadata(metadata, true);
    190        }
    191
    192        @Override
    193        public void addMetadata(MetadataType metadata) {
    194                addMetadata(metadata, false);
    195        }
    196
    197        private synchronized void addMetadata(MetadataType metadata, boolean clear) {
    198                if (metadata == null) {
    199                        return;
    200                }
    201
    202                if (this.metadata == null) {
    203                        this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    204                }
    205
    206                Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass());
    207                if (!this.metadata.containsKey(clazz)) {
    208                        this.metadata.put(clazz, new ArrayList<MetadataType>());
    209                } else if (clear) {
    210                        this.metadata.get(clazz).clear();
    211                }
    212                this.metadata.get(clazz).add(metadata);
    213
    214                // add for special case of sub-interfaces of IMetadata
    215                if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) {
    216                        clazz = IMetadata.class;
    217                        if (!this.metadata.containsKey(clazz)) {
    218                                this.metadata.put(clazz, new ArrayList<MetadataType>());
    219                        } else if (clear) {
    220                                this.metadata.get(clazz).clear();
    221                        }
    222                        this.metadata.get(clazz).add(metadata);
    223                }
    224        }
    225
    226        @Override
    227        @Deprecated
    228        public synchronized IMetadata getMetadata() {
    229                return getFirstMetadata(IMetadata.class);
    230        }
    231
    232        @SuppressWarnings("unchecked")
    233        @Override
    234        public synchronized <T extends MetadataType> List<T> getMetadata(Class<T> clazz) throws MetadataException {
    235                if (metadata == null) {
    236                        dirty = false;
    237                        return null;
    238                }
    239
    240                if (dirty) {
    241                        dirtyMetadata();
    242                        dirty = false;
    243                }
    244
    245                if (clazz == null) {
    246                        List<T> all = new ArrayList<>();
    247                        for (Class<? extends MetadataType> c : metadata.keySet()) {
    248                                all.addAll((Collection<T>) metadata.get(c));
    249                        }
    250                        return all;
    251                }
    252
    253                return (List<T>) metadata.get(findMetadataTypeSubInterfaces(clazz));
    254        }
    255
    256        @Override
    257        public synchronized <T extends MetadataType> T getFirstMetadata(Class<T> clazz) {
    258                try {
    259                        List<T> ml = getMetadata(clazz);
    260                        if (ml == null) {
    261                                return null;
    262                        }
    263                        for (T t : ml) {
    264                                if (clazz.isInstance(t)) {
    265                                        return t;
    266                                }
    267                        }
    268                } catch (Exception e) {
    269                        logger.error("Get metadata failed!",e);
    270                }
    271
    272                return null;
    273        }
    274
    275        @Override
    276        public synchronized void clearMetadata(Class<? extends MetadataType> clazz) {
    277                if (metadata == null) {
    278                        return;
    279                }
    280
    281                if (clazz == null) {
    282                        metadata.clear();
    283                        return;
    284                }
    285
    286                List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz));
    287                if( list != null) {
    288                        list.clear();
    289                }
    290        }
    291
    292        /**
    293         * @return copy of metadata
    294         * @since 2.0
    295         */
    296        protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() {
    297                return copyMetadata(metadata);
    298        }
    299
    300        /**
    301         * @param metadata type
    302         * @return copy of metadata of given type
    303         * @since 2.0
    304         */
    305        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) {
    306                if (metadata == null) {
    307                        return null;
    308                }
    309
    310                ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    311                copyMetadata(metadata, map);
    312                return map;
    313        }
    314
    315        private static void copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> inMetadata,
    316                        Map<Class<? extends MetadataType>, List<MetadataType>> outMetadata) {
    317                for (Class<? extends MetadataType> c : inMetadata.keySet()) {
    318                        List<MetadataType> l = inMetadata.get(c);
    319                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
    320                        outMetadata.put(c, nl);
    321                        for (MetadataType m : l) {
    322                                if (m == null || isMetadataDirty(m)) { // skip dirty metadata
    323                                        continue;
    324                                }
    325                                nl.add(m.clone());
    326                        }
    327                }
    328        }
    329
    330        protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) {
    331                copyMetadata(oldMetadata, metadata);
    332        }
    333
    334        /**
    335         * @param a dataset
    336         * @param clone if true, copy metadata
    337         * @return copy of metadata
    338         * @since 2.2
    339         */
    340        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> getMetadataMap(ILazyDataset a, boolean clone) {
    341                List<MetadataType> all = null;
    342                try {
    343                        all = a.getMetadata(null);
    344                } catch (Exception e) {
    345                }
    346                if (all == null) {
    347                        return null;
    348                }
    349
    350                ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    351
    352                for (MetadataType m : all) {
    353                        if (m == null || isMetadataDirty(m)) { // skip dirty metadata
    354                                continue;
    355                        }
    356                        Class<? extends MetadataType> c = findMetadataTypeSubInterfaces(m.getClass());
    357                        List<MetadataType> l = map.get(c);
    358                        if (l == null) {
    359                                l = new ArrayList<MetadataType>();
    360                                map.put(c, l);
    361                        }
    362                        if (clone) {
    363                                m = m.clone();
    364                        }
    365                        l.add(m);
    366                }
    367                return map;
    368        }
    369
    370        private static boolean isMetadataDirty(MetadataType m) {
    371                Class<? extends MetadataType> c = m.getClass();
    372                for (Field f : c.getDeclaredFields()) {
    373                        if (f.isAnnotationPresent(Dirtiable.class)) {
    374                                Class<?> t = f.getType();
    375                                if (t.equals(boolean.class) || t.equals(Boolean.class)) {
    376                                        try {
    377                                                f.setAccessible(true);
    378                                                Object o = f.get(m);
    379                                                if (o.equals(true)) {
    380                                                        return true;
    381                                                }
    382                                        } catch (Exception e) {
    383                                                logger.debug("Could not retrieve value of dirty variable: {}", c.getCanonicalName(), e);
    384                                        }
    385                                }
    386                        }
    387                }
    388
    389                return false;
    390        }
    391
    392        interface MetadatasetAnnotationOperation {
    393                /**
    394                 * Process value of given field
    395                 * <p>
    396                 * When the field is not a container then the returned value
    397                 * may replace the old value
    398                 * @param f given field
    399                 * @param o value of field
    400                 * @return transformed field
    401                 */
    402                Object processField(Field f, Object o);
    403
    404                /**
    405                 * @return annotated class
    406                 */
    407                Class<? extends Annotation> getAnnClass();
    408
    409                /**
    410                 * @param axis
    411                 * @return number of dimensions to insert or remove
    412                 */
    413                int change(int axis);
    414
    415                /**
    416                 *
    417                 * @return rank or -1 to match
    418                 */
    419                int getNewRank();
    420
    421                /**
    422                 * Run on given lazy dataset
    423                 * @param lz
    424                 * @return
    425                 */
    426                ILazyDataset run(ILazyDataset lz);
    427        }
    428
    429        class MdsSlice implements MetadatasetAnnotationOperation {
    430                private boolean asView;
    431                private SliceND slice;
    432                private int[] oShape;
    433                private long oSize;
    434
    435                public MdsSlice(boolean asView, SliceND slice) {
    436                        this.asView = asView;
    437                        this.slice = slice;
    438                        oShape = slice.getSourceShape();
    439                        oSize = ShapeUtils.calcLongSize(oShape);
    440                }
    441
    442                @Override
    443                public Object processField(Field field, Object o) {
    444                        return o;
    445                }
    446
    447                @Override
    448                public Class<? extends Annotation> getAnnClass() {
    449                        return Sliceable.class;
    450                }
    451
    452                @Override
    453                public int change(int axis) {
    454                        return 0;
    455                }
    456
    457                @Override
    458                public int getNewRank() {
    459                        return -1;
    460                }
    461
    462                @Override
    463                public ILazyDataset run(ILazyDataset lz) {
    464                        int rank = lz.getRank();
    465                        if (slice.getStart().length != rank) {
    466                                throw new IllegalArgumentException("Slice rank does not match dataset!");
    467                        }
    468
    469                        int[] shape = lz.getShape();
    470                        SliceND nslice;
    471                        if (!ShapeUtils.areShapesBroadcastCompatible(oShape, shape)) {
    472                                nslice = new SliceND(shape);
    473                                for (int i = 0; i < rank; i++) {
    474                                        int s = shape[i];
    475                                        int os = oShape[i];
    476                                        if (s >= os) {
    477                                                nslice.setSlice(i, 0, os, 1);
    478                                        } else if (s == 1) {
    479                                                nslice.setSlice(i, 0, 1, 1);
    480                                        } else {
    481                                                throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!");
    482                                        }
    483                                }
    484                                lz = lz.getSliceView(nslice);
    485                                shape = nslice.getShape();
    486                        }
    487                        if (lz.getSize() == oSize && Arrays.equals(shape, oShape)) {
    488                                nslice = slice;
    489                        } else {
    490                                nslice = slice.clone();
    491                                for (int i = 0; i < rank; i++) {
    492                                        int s = shape[i];
    493                                        if (s >= oShape[i]) {
    494                                                continue;
    495                                        } else if (s == 1) {
    496                                                nslice.setSlice(i, 0, 1, 1);
    497                                        } else {
    498                                                throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!");
    499                                        }
    500                                }
    501                                nslice.updateSourceShape(shape);
    502                        }
    503
    504                        if (asView || (lz instanceof IDataset)) {
    505                                return lz.getSliceView(nslice);
    506                        }
    507                        try {
    508                                return lz.getSlice(nslice);
    509                        } catch (DatasetException e) {
    510                                logger.error("Could not slice dataset in metadata", e);
    511                                return null;
    512                        }
    513                }
    514        }
    515
    516        class MdsReshape implements MetadatasetAnnotationOperation {
    517                private boolean matchRank;
    518                private int[] oldShape;
    519                private int[] newShape;
    520                boolean onesOnly;
    521                int[] differences;
    522
    523                /*
    524                 * if only ones then record differences (insertions and deletions)
    525                 *
    526                 * if shape changing, find broadcasted dimensions and disallow
    527                 * merging that include those dimensions
    528                 */
    529                public MdsReshape(final int[] oldShape, final int[] newShape) {
    530                        this.oldShape = oldShape;
    531                        this.newShape = newShape;
    532                        differences = null;
    533                }
    534
    535                @Override
    536                public Object processField(Field field, Object o) {
    537                        Annotation a = field.getAnnotation(Reshapeable.class);
    538                        if (a != null) { // cannot be null
    539                                matchRank = ((Reshapeable) a).matchRank();
    540                        }
    541                        return o;
    542                }
    543
    544                @Override
    545                public Class<? extends Annotation> getAnnClass() {
    546                        return Reshapeable.class;
    547                }
    548
    549                @Override
    550                public int change(int axis) {
    551                        if (matchRank) {
    552                                if (differences == null) {
    553                                        init();
    554                                }
    555
    556                                if (onesOnly) {
    557                                        return differences == null ? 0 : differences[axis];
    558                                }
    559                                throw new UnsupportedOperationException("TODO support other shape operations");
    560                        }
    561                        return 0;
    562                }
    563
    564                @Override
    565                public int getNewRank() {
    566                        return matchRank ? newShape.length : -1;
    567                }
    568
    569                private void init() {
    570                        int or = oldShape.length - 1;
    571                        int nr = newShape.length - 1;
    572                        if (or < 0 || nr < 0) { // zero-rank shapes
    573                                onesOnly = true;
    574                                differences = new int[1];
    575                                differences[0] = or < 0 ? nr + 1 : or + 1;
    576                                return;
    577                        }
    578                        onesOnly = ShapeUtils.differsByOnes(oldShape, newShape);
    579                        int ob = 0;
    580                        int nb = 0;
    581                        if (onesOnly) {
    582                                differences = ShapeUtils.calcShapePadding(oldShape, newShape);
    583                        } else {
    584                                differences = new int[or + 2];
    585                                if (matchRank) {
    586                                        logger.error("Combining dimensions is currently not supported");
    587                                        throw new IllegalArgumentException("Combining dimensions is currently not supported");
    588                                }
    589                                // work out mapping: contiguous dimensions can be grouped or split
    590                                while (ob <= or && nb <= nr) {
    591                                        int ol = oldShape[ob];
    592                                        while (ol == 1 && ol <= or) {
    593                                                ob++;
    594                                                ol = oldShape[ob];
    595                                        }
    596                                        int oe = ob + 1;
    597                                        int nl = newShape[nb];
    598                                        while (nl == 1 && nl <= nr) {
    599                                                nb++;
    600                                                nl = newShape[nb];
    601                                        }
    602                                        int ne = nb + 1;
    603                                        if (ol < nl) {
    604                                                differences[ob] = 1;
    605                                                do { // case where new shape combines several dimensions into one dimension
    606                                                        if (oe == (or + 1)) {
    607                                                                break;
    608                                                        }
    609                                                        differences[oe] = 1;
    610                                                        ol *= oldShape[oe++];
    611                                                } while (ol < nl);
    612                                                differences[oe - 1] = oe - ob; // signal end with difference
    613                                                if (nl != ol) {
    614                                                        logger.error("Single dimension is incompatible with subshape");
    615                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
    616                                                }
    617                                        } else if (ol > nl) {
    618                                                do { // case where new shape spreads single dimension over several dimensions
    619                                                        if (ne == (nr + 1)) {
    620                                                                break;
    621                                                        }
    622                                                        nl *= newShape[ne++];
    623                                                } while (nl < ol);
    624                                                if (nl != ol) {
    625                                                        logger.error("Subshape is incompatible with single dimension");
    626                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
    627                                                }
    628                                        }
    629
    630                                        ob = oe;
    631                                        nb = ne;
    632                                }
    633                        }
    634                }
    635
    636                @Override
    637                public ILazyDataset run(ILazyDataset lz) {
    638                        if (differences == null) {
    639                                init();
    640                        }
    641
    642                        int[] lshape = lz.getShape();
    643                        if (Arrays.equals(newShape, lshape)) {
    644                                return lz;
    645                        }
    646                        int or = lshape.length;
    647                        int nr = newShape.length;
    648                        int[] nshape;
    649                        if (onesOnly) {
    650                                nshape = ShapeUtils.padShape(differences, nr, lshape);
    651                        } else {
    652                                nshape = new int[nr];
    653                                boolean[] broadcast = new boolean[or];
    654                                for (int ob = 0; ob < or; ob++) {
    655                                        broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1;
    656                                }
    657                                int osize = lz.getSize();
    658
    659                                // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...)
    660                                int ob = 0;
    661                                int nsize = 1;
    662                                for (int i = 0; i < nr; i++) {
    663                                        if (ob < or && broadcast[ob]) {
    664                                                if (differences[ob] != 0) {
    665                                                        logger.error("Metadata contains a broadcast axis which cannot be reshaped");
    666                                                        throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped");
    667                                                }
    668                                                nshape[i] = 1;
    669                                        } else {
    670                                                nshape[i] = nsize < osize ? newShape[i] : 1;
    671                                        }
    672                                        nsize *= nshape[i];
    673                                        ob++;
    674                                }
    675                        }
    676
    677                        ILazyDataset nlz;
    678                        if (lz instanceof Dataset) {
    679                                nlz = ((Dataset) lz).reshape(nshape);
    680                        } else {
    681                                nlz = lz.getSliceView();
    682                                nlz.setShape(nshape);
    683                        }
    684                        return nlz;
    685                }
    686        }
    687
    688        class MdsTranspose implements MetadatasetAnnotationOperation {
    689                int[] map;
    690
    691                public MdsTranspose(final int[] axesMap) {
    692                        map = axesMap;
    693                }
    694
    695                @SuppressWarnings({ "rawtypes", "unchecked" })
    696                @Override
    697                public Object processField(Field f, Object o) {
    698                        // reorder arrays and lists according the axes map
    699                        if (o.getClass().isArray()) {
    700                                int l = Array.getLength(o);
    701                                if (l == map.length) {
    702                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
    703                                        for (int i = 0; i < l; i++) {
    704                                                Array.set(narray, i, Array.get(o, map[i]));
    705                                        }
    706                                        for (int i = 0; i < l; i++) {
    707                                                Array.set(o, i, Array.get(narray, i));
    708                                        }
    709                                }
    710                        } else if (o instanceof List<?>) {
    711                                List list = (List) o;
    712                                int l = list.size();
    713                                if (l == map.length) {
    714                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
    715                                        for (int i = 0; i < l; i++) {
    716                                                Array.set(narray, i, list.get(map[i]));
    717                                        }
    718                                        list.clear();
    719                                        for (int i = 0; i < l; i++) {
    720                                                list.add(Array.get(narray, i));
    721                                        }
    722                                }
    723                        }
    724                        return o;
    725                }
    726
    727                @Override
    728                public Class<? extends Annotation> getAnnClass() {
    729                        return Transposable.class;
    730                }
    731
    732                @Override
    733                public int change(int axis) {
    734                        return 0;
    735                }
    736
    737                @Override
    738                public int getNewRank() {
    739                        return -1;
    740                }
    741
    742                @Override
    743                public ILazyDataset run(ILazyDataset lz) {
    744                        return lz.getTransposedView(map);
    745                }
    746        }
    747
    748        class MdsDirty implements MetadatasetAnnotationOperation {
    749
    750                @Override
    751                public Object processField(Field f, Object o) {
    752                        // throw exception if not boolean???
    753                        Class<?> t = f.getType();
    754                        if (t.equals(boolean.class) || t.equals(Boolean.class)) {
    755                                if (o.equals(false)) {
    756                                        o = true;
    757                                }
    758                        }
    759                        return o;
    760                }
    761
    762                @Override
    763                public Class<? extends Annotation> getAnnClass() {
    764                        return Dirtiable.class;
    765                }
    766
    767                @Override
    768                public int change(int axis) {
    769                        return 0;
    770                }
    771
    772                @Override
    773                public int getNewRank() {
    774                        return -1;
    775                }
    776
    777                @Override
    778                public ILazyDataset run(ILazyDataset lz) {
    779                        return lz;
    780                }
    781        }
    782
    783        /**
    784         * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced
    785         * dataset after cloning the metadata
    786         * @param asView if true then just a view
    787         * @param slice an n-D slice
    788         */
    789        protected void sliceMetadata(boolean asView, final SliceND slice) {
    790                processAnnotatedMetadata(new MdsSlice(asView, slice));
    791        }
    792
    793        /**
    794         * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing
    795         * or setting the shape
    796         * @param oldShape old shape
    797         * @param newShape new shape
    798         */
    799        protected void reshapeMetadata(final int[] oldShape, final int[] newShape) {
    800                processAnnotatedMetadata(new MdsReshape(oldShape, newShape));
    801        }
    802
    803        /**
    804         * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed
    805         * dataset after cloning the metadata
    806         * @param axesMap if zero length then axes order reversed
    807         */
    808        protected void transposeMetadata(final int[] axesMap) {
    809                processAnnotatedMetadata(new MdsTranspose(axesMap));
    810        }
    811
    812        /**
    813         * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified
    814         * @since 2.0
    815         */
    816        protected void dirtyMetadata() {
    817                processAnnotatedMetadata(new MdsDirty());
    818        }
    819
    820        @SuppressWarnings("unchecked")
    821        private void processAnnotatedMetadata(MetadatasetAnnotationOperation op) {
    822                if (metadata == null)
    823                        return;
    824
    825                for (List<MetadataType> l : metadata.values()) {
    826                        for (MetadataType m : l) {
    827                                if (m == null) {
    828                                        continue;
    829                                }
    830
    831                                Class<? extends MetadataType> mc = m.getClass();
    832                                do { // iterate over super-classes
    833                                        processClass(op, m, mc);
    834                                        Class<?> sclazz = mc.getSuperclass();
    835                                        if (!MetadataType.class.isAssignableFrom(sclazz)) {
    836                                                break;
    837                                        }
    838                                        mc = (Class<? extends MetadataType>) sclazz;
    839                                } while (true);
    840                        }
    841                }
    842        }
    843
    844        @SuppressWarnings({ "unchecked", "rawtypes" })
    845        private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc) {
    846                for (Field f : mc.getDeclaredFields()) {
    847                        if (!f.isAnnotationPresent(op.getAnnClass()))
    848                                continue;
    849
    850                        try {
    851                                f.setAccessible(true);
    852                                Object o = f.get(m);
    853                                if (o == null) {
    854                                        continue;
    855                                }
    856
    857                                Object no = op.processField(f, o);
    858                                if (no != o) {
    859                                        f.set(m, no);
    860                                        continue;
    861                                }
    862                                Object r = null;
    863                                if (o instanceof ILazyDataset) {
    864                                        try {
    865                                                f.set(m, op.run((ILazyDataset) o));
    866                                        } catch (Exception e) {
    867                                                logger.error("Problem processing " + o, e);
    868                                                throw e;
    869                                        }
    870                                } else if (o.getClass().isArray()) {
    871                                        int l = Array.getLength(o);
    872
    873                                        for (int i = 0; r == null && i < l; i++) {
    874                                                r = Array.get(o, i);
    875                                        }
    876                                        int n = op.getNewRank();
    877                                        if (r == null) {
    878                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
    879                                                        f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n));
    880                                                }
    881                                                continue;
    882                                        }
    883                                        if (n < 0) {
    884                                                n = l;
    885                                        }
    886                                        Object narray = Array.newInstance(r.getClass(), n);
    887                                        for (int i = 0, si = 0, di = 0; di < n && si < l; i++) {
    888                                                int c = op.change(i);
    889                                                if (c == 0) {
    890                                                        Array.set(narray, di++, processObject(op, Array.get(o, si++)));
    891                                                } else if (c > 0) {
    892                                                        di += c; // add nulls by skipping forward in destination array
    893                                                } else if (c < 0) {
    894                                                        si -= c; // remove dimensions by skipping forward in source array
    895                                                }
    896                                        }
    897                                        if (n == l) {
    898                                                for (int i = 0; i < l; i++) {
    899                                                        Array.set(o, i, Array.get(narray, i));
    900                                                }
    901                                        } else {
    902                                                f.set(m, narray);
    903                                        }
    904                                } else if (o instanceof List<?>) {
    905                                        List list = (List) o;
    906                                        int l = list.size();
    907
    908                                        for (int i = 0; r == null && i < l; i++) {
    909                                                r = list.get(i);
    910                                        }
    911                                        int n = op.getNewRank();
    912                                        if (r == null) {
    913                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
    914                                                        list.clear();
    915                                                        for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) {
    916                                                                list.add(null);
    917                                                        }
    918                                                }
    919                                                continue;
    920                                        }
    921
    922                                        if (n < 0) {
    923                                                n = l;
    924                                        }
    925                                        Object narray = Array.newInstance(r.getClass(), n);
    926                                        for (int i = 0, si = 0, di = 0; i < l && si < l; i++) {
    927                                                int c = op.change(i);
    928                                                if (c == 0) {
    929                                                        Array.set(narray, di++, processObject(op, list.get(si++)));
    930                                                } else if (c > 0) {
    931                                                        di += c; // add nulls by skipping forward in destination array
    932                                                } else if (c < 0) {
    933                                                        si -= c; // remove dimensions by skipping forward in source array
    934                                                }
    935                                        }
    936                                        list.clear();
    937                                        for (int i = 0; i < n; i++) {
    938                                                list.add(Array.get(narray, i));
    939                                        }
    940                                } else if (o instanceof Map<?,?>) {
    941                                        Map map = (Map) o;
    942                                        for (Object k : map.keySet()) {
    943                                                map.put(k, processObject(op, map.get(k)));
    944                                        }
    945                                }
    946                        } catch (Exception e) {
    947                                logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e);
    948                                throw new RuntimeException(e);
    949                        }
    950                }
    951        }
    952
    953        @SuppressWarnings({ "unchecked", "rawtypes" })
    954        private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception {
    955                if (o == null) {
    956                        return o;
    957                }
    958
    959                if (o instanceof ILazyDataset) {
    960                        try {
    961                                return op.run((ILazyDataset) o);
    962                        } catch (Exception e) {
    963                                logger.error("Problem processing " + o, e);
    964                                throw e;
    965                        }
    966                } else if (o.getClass().isArray()) {
    967                        int l = Array.getLength(o);
    968                        for (int i = 0; i < l; i++) {
    969                                Array.set(o, i, processObject(op, Array.get(o, i)));
    970                        }
    971                } else if (o instanceof List<?>) {
    972                        List list = (List) o;
    973                        for (int i = 0, imax = list.size(); i < imax; i++) {
    974                                list.set(i, processObject(op, list.get(i)));
    975                        }
    976                } else if (o instanceof Map<?,?>) {
    977                        Map map = (Map) o;
    978                        for (Object k : map.keySet()) {
    979                                map.put(k, processObject(op, map.get(k)));
    980                        }
    981                }
    982                return o;
    983        }
    984
    985        protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) {
    986                ILazyDataset d = null;
    987                if (blob instanceof ILazyDataset) {
    988                        d = (ILazyDataset) blob;
    989                        if (d instanceof IDataset) {
    990                                Dataset ed = DatasetUtils.convertToDataset((IDataset) d);
    991                                int is = ed.getElementsPerItem();
    992                                if (is != 1 && is != getElementsPerItem()) {
    993                                        throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset");
    994                                }
    995                                d = ed.cast(is == 1 ? DoubleDataset.class: CompoundDoubleDataset.class);
    996                        } else if (!keepLazy) {
    997                                final int is = getElementsPerItem();
    998                                try {
    999                                        d = DatasetUtils.cast(is == 1 ? DoubleDataset.class: CompoundDoubleDataset.class, d.getSlice());
    1000                                } catch (DatasetException e) {
    1001                                        logger.error("Could not get data from lazy dataset", e);
    1002                                        return null;
    1003                                }
    1004                        }
    1005                } else {
    1006                        final int is = getElementsPerItem();
    1007                        if (is == 1) {
    1008                                d = DatasetFactory.createFromObject(DoubleDataset.class, blob);
    1009                        } else {
    1010                                try {
    1011                                        d = DatasetFactory.createFromObject(is, CompoundDoubleDataset.class, blob);
    1012                                } catch (IllegalArgumentException e) { // if only single value supplied try again
    1013                                        d = DatasetFactory.createFromObject(DoubleDataset.class, blob);
    1014                                }
    1015                        }
    1016                        if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) {
    1017                                d.setShape(shape.clone());
    1018                        }
    1019                }
    1020                List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape());
    1021                d.setShape(s.get(0));
    1022
    1023                return d;
    1024        }
    1025
    1026        @Override
    1027        public void setErrors(Serializable errors) {
    1028                if (shape == null) {
    1029                        throw new IllegalArgumentException("Cannot set errors for null dataset");
    1030                }
    1031                if (errors == null) {
    1032                        clearMetadata(ErrorMetadata.class);
    1033                        return;
    1034                }
    1035                if (errors == this) {
    1036                        logger.warn("Ignoring setting error to itself as this will lead to infinite recursion");
    1037                        return;
    1038                }
    1039
    1040                ILazyDataset errorData = createFromSerializable(errors, true);
    1041
    1042                ErrorMetadata emd = getErrorMetadata();
    1043                if (emd == null) {
    1044                        try {
    1045                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1046                                setMetadata(emd);
    1047                        } catch (MetadataException me) {
    1048                                logger.error("Could not create metadata", me);
    1049                        }
    1050                }
    1051                emd.setError(errorData);
    1052        }
    1053
    1054        protected ErrorMetadata getErrorMetadata() {
    1055                try {
    1056                        List<ErrorMetadata> el = getMetadata(ErrorMetadata.class);
    1057                        if (el != null && !el.isEmpty()) {
    1058                                 return el.get(0);
    1059                        }
    1060                } catch (Exception e) {
    1061                }
    1062                return null;
    1063        }
    1064
    1065        @Override
    1066        public ILazyDataset getErrors() {
    1067                ErrorMetadata emd = getErrorMetadata();
    1068                return emd == null ? null : emd.getError();
    1069        }
    1070
    1071        @Override
    1072        public boolean hasErrors() {
    1073                return LazyDatasetBase.this.getErrors() != null;
    1074        }
    1075
    1076        /**
    1077         * Check permutation axes
    1078         * @param shape to use
    1079         * @param axes if zero length then axes order reversed
    1080         * @return cleaned up copy of axes or null if trivial
    1081         */
    1082        public static int[] checkPermutatedAxes(int[] shape, int... axes) {
    1083                int rank = shape == null ? 0 : shape.length;
    1084
    1085                if (axes == null || axes.length == 0) {
    1086                        axes = new int[rank];
    1087                        for (int i = 0; i < rank; i++) {
    1088                                axes[i] = rank - 1 - i;
    1089                        }
    1090                } else {
    1091                        axes = axes.clone();
    1092                }
    1093
    1094                if (axes.length != rank) {
    1095                        logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank);
    1096                        throw new IllegalArgumentException("axis permutation does not match shape of dataset");
    1097                }
    1098
    1099                // check all permutation values are within bounds
    1100                for (int i = 0; i < rank; i++) {
    1101                        axes[i] = ShapeUtils.checkAxis(rank, axes[i]);
    1102                }
    1103
    1104                // check for a valid permutation (is this an unnecessary restriction?)
    1105                int[] perm = axes.clone();
    1106                Arrays.sort(perm);
    1107
    1108                for (int i = 0; i < rank; i++) {
    1109                        if (perm[i] != i) {
    1110                                logger.error("axis permutation is not valid: it does not contain complete set of axes");
    1111                                throw new IllegalArgumentException("axis permutation does not contain complete set of axes");
    1112                        }
    1113                }
    1114
    1115                if (Arrays.equals(axes, perm)) {
    1116                        return null; // signal identity or trivial permutation
    1117                }
    1118
    1119                return axes;
    1120        }
    1121}