support for displaying annotations on videos

LICENSE

+This project is dual-licensed under the Apache License 2.0 and the MIT license.
+
+Copyright 2021 Digital Humanities Initiative, Center for Evolving Humanities, Graduate School of Humanities and Sociology, The University of Tokyo
+Copyright 2021 International Institute for Digital Humanities
+Copyright 2021 Research Institute for Languages and Cultures of Asia and Africa, Tokyo University of Foreign Studies
+Copyright 2021 FLX Style
+
+Includes content from Mirador licensed under the Apache License 2.0.
+
 Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
 
 Licensed under the Apache License, Version 2.0 (the "License");

README.md

+## Mirador with support for displaying annotations on videos
+
+### Project
+https://dh.l.u-tokyo.ac.jp/activity/iiif/video-annotation
+
+### Demo
+- https://dzkimgs.l.u-tokyo.ac.jp/videos/m3/cat_video.html
+- https://dzkimgs.l.u-tokyo.ac.jp/videos/m3/video.html
+
+### Manifest Sample
+- https://dzkimgs.l.u-tokyo.ac.jp/videos/cat2020/manifest.json
+- https://dzkimgs.l.u-tokyo.ac.jp/videos/iiif_in_japan_2017/manifest.json
+
+### Prebuilt
+https://dzkimgs.l.u-tokyo.ac.jp/videos/m3/mirador.min.js
+
+### License
+This project is dual-licensed under the Apache License 2.0 and the MIT license. See [LICENSE](LICENSE) for details.
+
+---
 *NOTE: This README reflects the latest version of Mirador, Mirador 3. For previous versions, please reference that release's README directly. Latest 2.x release: [v.2.7.0](https://github.com/ProjectMirador/mirador/tree/v2.7.0)*
 # Mirador
 ![Node.js CI](https://github.com/ProjectMirador/mirador/workflows/Node.js%20CI/badge.svg) [![codecov](https://codecov.io/gh/ProjectMirador/mirador/branch/master/graph/badge.svg)](https://codecov.io/gh/ProjectMirador/mirador) 

src/components/VideoViewer.js

-import React, { Component, Fragment } from 'react';
+import flatten from 'lodash/flatten';
+import flattenDeep from 'lodash/flattenDeep';
+import React, { Component } from 'react';
 import PropTypes from 'prop-types';
+import AnnotationItem from '../lib/AnnotationItem';
+import AnnotationsOverlayVideo from '../containers/AnnotationsOverlayVideo';
+import WindowCanvasNavigationControlsVideo from '../containers/WindowCanvasNavigationControlsVideo';
 
 /** */
 export class VideoViewer extends Component {
+  /** */
+  constructor(props) {
+    super(props);
+    this.videoRef = React.createRef();
+
+    this.state = {
+      start: 0,
+      time: 0,
+    };
+  }
+
+  /** */
+  componentDidUpdate(prevProps) {
+    const {
+      canvas, currentTime, muted, paused,
+      setCurrentTime, setPaused,
+    } = this.props;
+
+    if (paused !== prevProps.paused) {
+      if (currentTime === 0) {
+        this.timerReset();
+      }
+      if (paused) {
+        this.timerStop();
+      } else {
+        this.timerStart();
+      }
+    }
+    if (currentTime !== prevProps.currentTime) {
+      const duration = canvas.getDuration();
+      if (duration && duration < currentTime) {
+        if (!paused) {
+          setPaused(true);
+          setCurrentTime(0);
+          this.timerReset();
+        }
+      }
+    }
+    const video = this.videoRef.current;
+    if (video) {
+      if (video.muted !== muted) {
+        video.muted = muted;
+      }
+    }
+  }
+
+  /** */
+  timerStart() {
+    const { currentTime } = this.props;
+    this.setState({
+      start: Date.now() - currentTime * 1000,
+      time: currentTime * 1000,
+    });
+    this.timer = setInterval(() => {
+      const { setCurrentTime } = this.props;
+      this.setState(prevState => ({
+        time: Date.now() - prevState.start,
+      }));
+      const { time } = this.state;
+      setCurrentTime(time / 1000);
+    }, 100);
+  }
+
+  /** */
+  timerStop() {
+    clearInterval(this.timer);
+  }
+
+  /** */
+  timerReset() {
+    this.setState({ time: 0 });
+  }
+
   /* eslint-disable jsx-a11y/media-has-caption */
   /** */
   render() {
-    const { captions, classes, videoResources } = this.props;
+    const {
+      canvas, classes, currentTime, windowId,
+    } = this.props;
+
+    const videoResources = flatten(
+      flattenDeep([
+        canvas.getContent().map(annot => {
+          const annotaion = new AnnotationItem(annot.__jsonld);
+          const temporalfragment = annotaion.temporalfragmentSelector;
+          if (temporalfragment && temporalfragment.length > 0) {
+            const start = temporalfragment[0] || 0;
+            const end = (temporalfragment.length > 1) ? temporalfragment[1] : Number.MAX_VALUE;
+            if (start <= currentTime && currentTime < end) {
+              //
+            } else {
+              return {};
+            }
+          }
+          const body = annot.getBody();
+          return { body, temporalfragment };
+        }),
+      ]).filter((resource) => resource.body && resource.body[0].__jsonld && resource.body[0].__jsonld.type === 'Video'),
+    );
+    // Only one video can be displayed at a time in this implementation.
+    const len = videoResources.length;
+    const video = len > 0
+      ? videoResources[len - 1].body[0] : null;
+    const videoTargetTemporalfragment = len > 0
+      ? videoResources[len - 1].temporalfragment : [];
+
     return (
-      <div className={classes.container}>
-        <video controls className={classes.video}>
-          {videoResources.map(video => (
-            <Fragment key={video.id}>
+      <div className={classes.flexContainer}>
+        <div className={classes.flexFill}>
+          { video && (
+            <>
+              <video className={classes.video} key={video.id} ref={this.videoRef}>
                 <source src={video.id} type={video.getFormat()} />
-            </Fragment>
-          ))}
-          {captions.map(caption => (
-            <Fragment key={caption.id}>
-              <track src={caption.id} label={caption.getLabel()} srcLang={caption.getProperty('language')} />
-            </Fragment>
-          ))}
               </video>
+              <AnnotationsOverlayVideo windowId={windowId} videoRef={this.videoRef} videoTarget={videoTargetTemporalfragment} key={`${windowId} ${video.id}`} />
+            </>
+          )}
+          <WindowCanvasNavigationControlsVideo windowId={windowId} />
+        </div>
       </div>
     );
   }
@@ -28,12 +134,21 @@ export class VideoViewer extends Component {
 }
 
 VideoViewer.propTypes = {
-  captions: PropTypes.arrayOf(PropTypes.object),
+  canvas: PropTypes.object, // eslint-disable-line react/forbid-prop-types
   classes: PropTypes.objectOf(PropTypes.string).isRequired,
-  videoResources: PropTypes.arrayOf(PropTypes.object),
+  currentTime: PropTypes.number,
+  muted: PropTypes.bool,
+  paused: PropTypes.bool,
+  setCurrentTime: PropTypes.func,
+  setPaused: PropTypes.func,
+  windowId: PropTypes.string.isRequired,
 };
 
 VideoViewer.defaultProps = {
-  captions: [],
-  videoResources: [],
+  canvas: {},
+  currentTime: 0,
+  muted: false,
+  paused: true,
+  setCurrentTime: () => {},
+  setPaused: () => {},
 };

src/components/ViewerNavigation.js

@@ -15,6 +15,7 @@ export class ViewerNavigation extends Component {
     const {
       hasNextCanvas, hasPreviousCanvas, setNextCanvas, setPreviousCanvas, t,
       classes, viewingDirection,
+      beforeClick,
     } = this.props;
 
     let htmlDir = 'ltr';
@@ -48,7 +49,7 @@ export class ViewerNavigation extends Component {
           aria-label={t('previousCanvas')}
           className={ns('previous-canvas-button')}
           disabled={!hasPreviousCanvas}
-          onClick={() => { hasPreviousCanvas && setPreviousCanvas(); }}
+          onClick={() => { beforeClick(); hasPreviousCanvas && setPreviousCanvas(); }}
         >
           <NavigationIcon style={previousIconStyle} />
         </MiradorMenuButton>
@@ -56,7 +57,7 @@ export class ViewerNavigation extends Component {
           aria-label={t('nextCanvas')}
           className={ns('next-canvas-button')}
           disabled={!hasNextCanvas}
-          onClick={() => { hasNextCanvas && setNextCanvas(); }}
+          onClick={() => { beforeClick(); hasNextCanvas && setNextCanvas(); }}
         >
           <NavigationIcon style={nextIconStyle} />
         </MiradorMenuButton>
@@ -66,6 +67,7 @@ export class ViewerNavigation extends Component {
 }
 
 ViewerNavigation.propTypes = {
+  beforeClick: PropTypes.func,
   classes: PropTypes.objectOf(PropTypes.string).isRequired,
   hasNextCanvas: PropTypes.bool,
   hasPreviousCanvas: PropTypes.bool,
@@ -76,6 +78,7 @@ ViewerNavigation.propTypes = {
 };
 
 ViewerNavigation.defaultProps = {
+  beforeClick: () => {},
   hasNextCanvas: false,
   hasPreviousCanvas: false,
   setNextCanvas: () => {},

src/components/ViewerNavigationVideo.js

+import React, { Component } from 'react';
+import PauseRoundedIcon from '@material-ui/icons/PauseRounded';
+import PlayArrowRoundedIcon from '@material-ui/icons/PlayArrowRounded';
+import PropTypes from 'prop-types';
+import Slider from '@material-ui/core/Slider';
+import Typography from '@material-ui/core/Typography';
+import VolumeOffIcon from '@material-ui/icons/VolumeOff';
+import VolumeUpIcon from '@material-ui/icons/VolumeUp';
+import MiradorMenuButton from '../containers/MiradorMenuButton';
+import ns from '../config/css-ns';
+
+/** ViewerNavigationVideo - based on ViewerNavigation */
+export class ViewerNavigationVideo extends Component {
+  /** */
+  constructor(props) {
+    super(props);
+    this.handleChange = this.handleChange.bind(this);
+  }
+
+  /** */
+  handleChange = (event, newValue) => {
+    const { paused, setCurrentTime, setPaused } = this.props;
+    if (!paused) {
+      // In the flow of pausing, adjusting currentTime, and resuming playback,
+      // it is necessary to handle cases where the user explicitly pauses playback.
+      setPaused(true);
+      setCurrentTime(newValue);
+      setPaused(false);
+    } else {
+      setCurrentTime(newValue);
+    }
+  };
+
+  /**
+   * Renders things
+   */
+  render() {
+    const {
+      classes,
+      currentTime,
+      duration,
+      muted,
+      paused,
+      setMuted,
+      setPaused,
+    } = this.props;
+
+    const start = (duration > 3600 || duration === undefined) ? 11 : 14;
+    const len = (duration > 3600 || duration === undefined) ? 8 : 5;
+    let durationLabel = new Date(currentTime * 1000).toISOString().substr(start, len);
+    let slider = '';
+    if (duration !== undefined) {
+      durationLabel = `${durationLabel} / ${new Date(duration * 1000).toISOString().substr(start, len)}`;
+      slider = (
+        <div className={classes.sliderDiv}>
+          <Slider value={currentTime} min={0} max={duration} onChange={this.handleChange} />
+        </div>
+      );
+    }
+    return (
+      <div className={classes.play_controls}>
+        <MiradorMenuButton
+          aria-label={paused ? 'Play' : 'Pause'}
+          className={paused ? ns('next-canvas-button') : ns('next-canvas-button')}
+          onClick={() => { setPaused(!paused); }}
+        >
+          { paused ? <PlayArrowRoundedIcon /> : <PauseRoundedIcon /> }
+        </MiradorMenuButton>
+        {slider}
+        <span className={classes.timeLabel}>
+          <Typography variant="caption">
+            {durationLabel}
+          </Typography>
+        </span>
+        <MiradorMenuButton
+          aria-label={muted ? 'Unmute' : 'Mute'}
+          className={muted ? ns('next-canvas-button') : ns('next-canvas-button')}
+          onClick={() => { setMuted(!muted); }}
+        >
+          { muted ? <VolumeOffIcon /> : <VolumeUpIcon /> }
+        </MiradorMenuButton>
+        <span className={classes.divider} />
+      </div>
+    );
+  }
+}
+
+ViewerNavigationVideo.propTypes = {
+  classes: PropTypes.objectOf(PropTypes.string).isRequired,
+  currentTime: PropTypes.number,
+  duration: PropTypes.number,
+  muted: PropTypes.bool,
+  paused: PropTypes.bool,
+  setCurrentTime: PropTypes.func,
+  setMuted: PropTypes.func,
+  setPaused: PropTypes.func,
+};
+
+ViewerNavigationVideo.defaultProps = {
+  currentTime: 0,
+  duration: undefined,
+  muted: false,
+  paused: true,
+  setCurrentTime: () => {},
+  setMuted: () => {},
+  setPaused: () => {},
+};

src/components/WindowCanvasNavigationControlsVideo.js

+import React, { Component } from 'react';
+import PropTypes from 'prop-types';
+import classNames from 'classnames';
+import Paper from '@material-ui/core/Paper';
+import Typography from '@material-ui/core/Typography';
+import ViewerInfo from '../containers/ViewerInfo';
+import ViewerNavigation from '../containers/ViewerNavigation';
+import ViewerNavigationVideo from '../containers/ViewerNavigationVideo';
+import ns from '../config/css-ns';
+import { PluginHook } from './PluginHook';
+
+/**
+ * WindowCanvasNavigationControlsVideo - based on WindowCanvasNavigationControls
+ * Represents the viewer controls in the mirador workspace.
+ */
+export class WindowCanvasNavigationControlsVideo extends Component {
+  /**
+   * Determine if canvasNavControls are stacked (based on a hard-coded width)
+  */
+  canvasNavControlsAreStacked() {
+    const { size } = this.props;
+
+    return (size && size.width && size.width <= 253);
+  }
+
+  /** */
+  render() {
+    const {
+      classes, visible, windowId, setPaused,
+    } = this.props;
+
+    if (!visible) return (<Typography variant="srOnly" component="div"><ViewerInfo windowId={windowId} /></Typography>);
+
+    return (
+      <Paper
+        square
+        className={
+          classNames(
+            classes.controls,
+            ns('canvas-nav'),
+            classes.canvasNav,
+            this.canvasNavControlsAreStacked() ? ns('canvas-nav-stacked') : null,
+            this.canvasNavControlsAreStacked() ? classes.canvasNavStacked : null,
+          )
+        }
+        elevation={0}
+      >
+
+        <ViewerNavigation windowId={windowId} beforeClick={setPaused} />
+        <ViewerInfo windowId={windowId} />
+        <ViewerNavigationVideo windowId={windowId} />
+
+        <PluginHook {...this.props} />
+      </Paper>
+    );
+  }
+}
+
+WindowCanvasNavigationControlsVideo.propTypes = {
+  classes: PropTypes.objectOf(PropTypes.string),
+  setPaused: PropTypes.func,
+  size: PropTypes.shape({ width: PropTypes.number }).isRequired,
+  visible: PropTypes.bool,
+  windowId: PropTypes.string.isRequired,
+};
+
+WindowCanvasNavigationControlsVideo.defaultProps = {
+  classes: {},
+  setPaused: () => {},
+  visible: true,
+};

src/containers/AnnotationsOverlayVideo.js

+/** AnnotationsOverlayVideo - based on AnnotationsOverlay */
+import { compose } from 'redux';
+import { connect } from 'react-redux';
+import { withTranslation } from 'react-i18next';
+import { withPlugins } from '../extend/withPlugins';
+import { AnnotationsOverlayVideo } from '../components/AnnotationsOverlayVideo';
+import * as actions from '../state/actions';
+import {
+  getWindow,
+  getSearchAnnotationsForWindow,
+  getCompanionWindowsForContent,
+  getTheme,
+  getConfig,
+  getCurrentCanvas,
+  getWindowCurrentTime,
+  getWindowPausedStatus,
+  getPresentAnnotationsOnSelectedCanvases,
+  getSelectedAnnotationId,
+  getCurrentCanvasWorld,
+} from '../state/selectors';
+
+/**
+ * mapStateToProps - used to hook up connect to action creators
+ * @memberof Window
+ * @private
+ */
+const mapStateToProps = (state, { windowId }) => ({
+  annotations: getPresentAnnotationsOnSelectedCanvases(state, { windowId }),
+  canvas: (getCurrentCanvas(state, { windowId }) || {}),
+  canvasWorld: getCurrentCanvasWorld(state, { windowId }),
+  currentTime: getWindowCurrentTime(state, { windowId }),
+  drawAnnotations: getConfig(state).window.forceDrawAnnotations || getCompanionWindowsForContent(state, { content: 'annotations', windowId }).length > 0,
+  drawSearchAnnotations: getConfig(state).window.forceDrawAnnotations || getCompanionWindowsForContent(state, { content: 'search', windowId }).length > 0,
+  highlightAllAnnotations: getWindow(state, { windowId }).highlightAllAnnotations,
+  hoveredAnnotationIds: getWindow(state, { windowId }).hoveredAnnotationIds,
+  palette: getTheme(state).palette,
+  paused: getWindowPausedStatus(state, { windowId }),
+  searchAnnotations: getSearchAnnotationsForWindow(
+    state,
+    { windowId },
+  ),
+  selectedAnnotationId: getSelectedAnnotationId(state, { windowId }),
+});
+
+/**
+ * mapDispatchToProps - used to hook up connect to action creators
+ * @memberof ManifestListItem
+ * @private
+ */
+const mapDispatchToProps = (dispatch, { windowId }) => ({
+  deselectAnnotation: (...args) => dispatch(actions.deselectAnnotation(...args)),
+  hoverAnnotation: (...args) => dispatch(actions.hoverAnnotation(...args)),
+  selectAnnotation: (...args) => dispatch(actions.selectAnnotation(...args)),
+  setCurrentTime: (...args) => dispatch(actions.setWindowCurrentTime(windowId, ...args)),
+  setPaused: (...args) => dispatch(actions.setWindowPaused(windowId, ...args)),
+});
+
+const enhance = compose(
+  withTranslation(),
+  connect(mapStateToProps, mapDispatchToProps),
+  withPlugins('AnnotationsOverlayVideo'),
+);
+
+export default enhance(AnnotationsOverlayVideo);

src/containers/VideoViewer.js

@@ -3,26 +3,49 @@ import { compose } from 'redux';
 import { withTranslation } from 'react-i18next';
 import { withStyles } from '@material-ui/core';
 import { withPlugins } from '../extend/withPlugins';
+import * as actions from '../state/actions';
 import { VideoViewer } from '../components/VideoViewer';
-import { getVisibleCanvasCaptions, getVisibleCanvasVideoResources } from '../state/selectors';
+import {
+  getCurrentCanvas,
+  getCurrentCanvasWorld,
+  getWindowMutedStatus,
+  getWindowPausedStatus,
+  getWindowCurrentTime,
+} from '../state/selectors';
 
 /** */
-const mapStateToProps = (state, { windowId }) => (
-  {
-    captions: getVisibleCanvasCaptions(state, { windowId }) || [],
-    videoResources: getVisibleCanvasVideoResources(state, { windowId }) || [],
-  }
-);
+const mapStateToProps = (state, { windowId }) => ({
+  canvas: (getCurrentCanvas(state, { windowId }) || {}),
+  canvasWorld: getCurrentCanvasWorld(state, { windowId }),
+  currentTime: getWindowCurrentTime(state, { windowId }),
+  muted: getWindowMutedStatus(state, { windowId }),
+  paused: getWindowPausedStatus(state, { windowId }),
+});
+
+/** */
+const mapDispatchToProps = (dispatch, { windowId }) => ({
+  setCurrentTime: (...args) => dispatch(actions.setWindowCurrentTime(windowId, ...args)),
+  setPaused: (...args) => dispatch(actions.setWindowPaused(windowId, ...args)),
+});
 
 /** */
 const styles = () => ({
-  container: {
+  flexContainer: {
     alignItems: 'center',
     display: 'flex',
     width: '100%',
   },
+  flexFill: {
+    height: '100%',
+    position: 'relative',
+    width: '100%',
+  },
   video: {
+    height: '100%',
     maxHeight: '100%',
+    maxWidth: '100%',
+    'object-fit': 'scale-down',
+    'object-position': 'left top',
     width: '100%',
   },
 });
@@ -30,7 +53,7 @@ const styles = () => ({
 const enhance = compose(
   withTranslation(),
   withStyles(styles),
-  connect(mapStateToProps, null),
+  connect(mapStateToProps, mapDispatchToProps),
   withPlugins('VideoViewer'),
 );
 

src/containers/ViewerNavigationVideo.js

+import { compose } from 'redux';
+import { connect } from 'react-redux';
+import { withTranslation } from 'react-i18next';
+import { withStyles } from '@material-ui/core/styles';
+import { withPlugins } from '../extend/withPlugins';
+import * as actions from '../state/actions';
+import { ViewerNavigationVideo } from '../components/ViewerNavigationVideo';
+import {
+  getCurrentCanvasDuration,
+  getWindowCurrentTime,
+  getWindowMutedStatus,
+  getWindowPausedStatus,
+} from '../state/selectors';
+
+/** */
+const mapStateToProps = (state, { windowId }) => ({
+  currentTime: getWindowCurrentTime(state, { windowId }),
+  duration: getCurrentCanvasDuration(state, { windowId }),
+  muted: getWindowMutedStatus(state, { windowId }),
+  paused: getWindowPausedStatus(state, { windowId }),
+});
+
+/**
+ * mapDispatchToProps - used to hook up connect to action creators
+ * @memberof ManifestForm
+ * @private
+ */
+const mapDispatchToProps = (dispatch, { windowId }) => ({
+  setCurrentTime: (...args) => dispatch(actions.setWindowCurrentTime(windowId, ...args)),
+  setMuted: (...args) => dispatch(actions.setWindowMuted(windowId, ...args)),
+  setPaused: (...args) => dispatch(actions.setWindowPaused(windowId, ...args)),
+});
+
+const styles = {
+  divider: {
+    borderRight: '1px solid #808080',
+    display: 'inline-block',
+    height: '24px',
+    margin: '12px 6px',
+  },
+  ListItem: {
+    paddingBottom: 0,
+    paddingTop: 0,
+  },
+  play_controls: {
+    display: 'flex',
+    flexDirection: 'row',
+    justifyContent: 'center',
+  },
+  sliderDiv: {
+    alignItems: 'center',
+    display: 'flex',
+    paddingLeft: '10px',
+    paddingRight: '15px',
+    width: '200px',
+  },
+  timeLabel: {
+    alignItems: 'center',
+    display: 'flex',
+  },
+};
+
+const enhance = compose(
+  withStyles(styles),
+  withTranslation(),
+  connect(mapStateToProps, mapDispatchToProps),
+  withPlugins('ViewerNavigationVideo'),
+  // further HOC go here
+);
+
+export default enhance(ViewerNavigationVideo);

src/containers/WindowCanvasNavigationControlsVideo.js

+import { connect } from 'react-redux';
+import { compose } from 'redux';
+import { withSize } from 'react-sizeme';
+import { withStyles } from '@material-ui/core';
+import { fade } from '@material-ui/core/styles/colorManipulator';
+import { withPlugins } from '../extend/withPlugins';
+import { getWorkspace } from '../state/selectors';
+import { WindowCanvasNavigationControlsVideo } from '../components/WindowCanvasNavigationControlsVideo';
+import * as actions from '../state/actions';
+
+/** */
+const mapStateToProps = (state, { windowId }) => ({
+  visible: getWorkspace(state).focusedWindowId === windowId,
+});
+
+/** */
+const mapDispatchToProps = (dispatch, { windowId }) => ({
+  setPaused: (...args) => dispatch(actions.setWindowPaused(windowId, ...args)),
+});
+
+/**
+ *
+ * @param theme
+ */
+const styles = theme => ({
+  canvasNav: {
+    display: 'flex',
+    flexDirection: 'row',
+    flexWrap: 'wrap',
+    justifyContent: 'center',
+    textAlign: 'center',
+  },
+  canvasNavStacked: {
+    flexDirection: 'column',
+  },
+  controls: {
+    backgroundColor: fade(theme.palette.background.paper, 0.5),
+    bottom: 0,
+    position: 'absolute',
+    width: '100%',
+    zIndex: 50,
+  },
+});
+
+const enhance = compose(
+  connect(mapStateToProps),
+  withStyles(styles),
+  withSize(),
+  connect(mapStateToProps, mapDispatchToProps),
+  withPlugins('WindowCanvasNavigationControlsVideo'),
+);
+
+export default enhance(WindowCanvasNavigationControlsVideo);

src/lib/AnnotationItem.js

@@ -27,7 +27,7 @@ export default class AnnotationItem {
     const target = this.target[0];
     switch (typeof target) {
       case 'string':
-        return target.replace(/#?xywh=(.*)$/, '');
+        return target.replace(/#(.*)$/, '');
       case 'object':
         return (target.source && target.source.id) || target.source || target.id;
       default:
@@ -106,16 +106,50 @@ export default class AnnotationItem {
 
     switch (typeof selector) {
       case 'string':
-        match = selector.match(/xywh=(.*)$/);
+        match = selector.match(/xywh=(.*?)(&|$)/);
         break;
       case 'object':
         fragmentSelector = selector.find(s => s.type && s.type === 'FragmentSelector');
-        match = fragmentSelector && fragmentSelector.value.match(/xywh=(.*)$/);
+        match = fragmentSelector && fragmentSelector.value.match(/xywh=(.*?)(&|$)/);
         break;
       default:
         return null;
     }
 
-    return match && match[1].split(',').map(str => parseInt(str, 10));
+    if (match) {
+      const params = match[1].split(',');
+      if (params.length === 4) {
+        return params.map(str => parseInt(str, 10));
+      }
+    }
+    return null;
+  }
+
+  /** */
+  get temporalfragmentSelector() {
+    const { selector } = this;
+
+    let match;
+    let temporalfragmentSelector;
+
+    switch (typeof selector) {
+      case 'string':
+        match = selector.match(/t=(.*?)(&|$)/);
+        break;
+      case 'object':
+        temporalfragmentSelector = selector.find(s => s.type && s.type === 'FragmentSelector');
+        match = temporalfragmentSelector && temporalfragmentSelector.value.match(/t=(.*?)(&|$)/);
+        break;
+      default:
+        return null;
+    }
+
+    if (match) {
+      const params = match[1].split(',');
+      if (params.length < 3) {
+        return params.map(str => parseFloat(str));
+      }
+    }
+    return null;
   }
 }

src/lib/AnnotationResource.js

@@ -28,9 +28,9 @@ export default class AnnotationResource {
     const on = this.on[0];
     switch (typeof on) {
       case 'string':
-        return on.replace(/#?xywh=(.*)$/, '');
+        return on.replace(/#(.*)$/, '');
       case 'object':
-        return on.full.replace(/#?xywh=(.*)$/, '');
+        return on.full.replace(/#(.*)$/, '');
       default:
         return null;
     }
@@ -109,15 +109,47 @@ export default class AnnotationResource {
 
     switch (typeof selector) {
       case 'string':
-        match = selector.match(/xywh=(.*)$/);
+        match = selector.match(/xywh=(.*?)(&|$)/);
         break;
       case 'object':
-        match = selector.value.match(/xywh=(.*)$/);
+        match = selector.value.match(/xywh=(.*?)(&|$)/);
         break;
       default:
         return null;
     }
 
-    return match && match[1].split(',').map(str => parseInt(str, 10));
+    if (match) {
+      const params = match[1].split(',');
+      if (params.length === 4) {
+        return params.map(str => parseInt(str, 10));
+      }
+    }
+    return null;
+  }
+
+  /** */
+  get temporalfragmentSelector() {
+    const { selector } = this;
+
+    let match;
+
+    switch (typeof selector) {
+      case 'string':
+        match = selector.match(/t=(.*?)(&|$)/);
+        break;
+      case 'object':
+        match = selector.value.match(/t=(.*?)(&|$)/);
+        break;
+      default:
+        return null;
+    }
+
+    if (match) {
+      const params = match[1].split(',');
+      if (params.length < 3) {
+        return params.map(str => parseFloat(str));
+      }
+    }
+    return null;
   }
 }

src/lib/CanvasAnnotationDisplay.js

@@ -5,7 +5,7 @@
 export default class CanvasAnnotationDisplay {
   /** */
   constructor({
-    resource, palette, zoomRatio, offset, selected, hovered,
+    resource, palette, zoomRatio, offset, selected, hovered, imageSource, canvasSize,
   }) {
     this.resource = resource;
     this.palette = palette;
@@ -13,6 +13,8 @@ export default class CanvasAnnotationDisplay {
     this.offset = offset;
     this.selected = selected;
     this.hovered = hovered;
+    this.imageSource = imageSource;
+    this.canvasSize = canvasSize;
   }
 
   /** */
@@ -104,7 +106,8 @@ export default class CanvasAnnotationDisplay {
 
   /** */
   fragmentContext() {
-    const fragment = this.resource.fragmentSelector;
+    const fragment = this.resource.fragmentSelector || this.canvasSize;
+    if (!fragment) { return; }
     fragment[0] += this.offset.x;
     fragment[1] += this.offset.y;
 
@@ -131,6 +134,10 @@ export default class CanvasAnnotationDisplay {
       this.context.strokeRect(...fragment);
     }
 
+    if (this.imageSource) {
+      this.context.drawImage(this.imageSource, ...fragment);
+    }
+
     this.context.restore();
   }
 

src/lib/CanvasOverlayVideo.js

+/**
+ * CanvasOverlayVideo - based on the framework of OpenSeadragonCanvasOverlay
+ *
+ * OpenSeadragonCanvasOverlay - adapted from https://github.com/altert/OpenSeadragonCanvasOverlay
+ * Code ported from https://github.com/altert/OpenSeadragonCanvasOverlay
+ * carries a BSD 3-Clause license originally authored by @altert from
+ * https://github.com/altert/OpenseadragonFabricjsOverlay
+ */
+export default class CanvasOverlayVideo {
+  /**
+   * constructor - sets up the Canvas overlay container
+   * @param canvasSize {Array} IIIF Canvas size
+   * If the width and height properties are not specified in the Canvas in the Manifest,
+   * canvasSize will be [0, 0, 0, 0].
+   */
+  constructor(video, ref, canvasSize) {
+    this.video = video;
+    this.ref = ref;
+    const [
+      _canvasX, _canvasY, canvasWidth, canvasHeight, // eslint-disable-line no-unused-vars
+    ] = canvasSize;
+    this.canvasWidth = canvasWidth;
+    this.canvasHeight = canvasHeight;
+
+    this.containerWidth = 0;
+    this.containerHeight = 0;
+  }
+
+  /** */
+  get canvas() {
+    return this.ref.current;
+  }
+
+  /** */
+  get context2d() {
+    return this.canvas ? this.canvas.getContext('2d') : null;
+  }
+
+  /**
+   * scale - get the display scaling factor of the HTML5 canvas.
+   * It is assumed that the size of the Canvas in the Manifest is equal to the size of the video.
+   * This will not work correctly if multiple videos are placed on the Canvas.
+   */
+  get scale() {
+    let ratio = 1;
+
+    if (this.video) {
+      const { videoWidth, videoHeight } = this.video;
+      if (videoWidth && videoHeight) {
+        const ratioWidth = this.containerWidth / videoWidth;
+        const rationHeight = this.containerHeight / videoHeight;
+        ratio = Math.min(ratioWidth, rationHeight);
+        if (ratio > 1) {
+          const objectFit = getComputedStyle(this.video, null).getPropertyValue('object-fit');
+          if (objectFit === 'scale-down' || objectFit === 'none') {
+            ratio = 1;
+          }
+        }
+      }
+    } else if (this.canvasWidth && this.canvasHeight) {
+      // video is not loaded yet & canvas size is specified
+      const ratioWidth = this.containerWidth / this.canvasWidth;
+      const rationHeight = this.containerHeight / this.canvasHeight;
+      ratio = Math.min(ratioWidth, rationHeight);
+      if (ratio > 1) {
+        ratio = 1;
+      }
+    }
+
+    return ratio;
+  }
+
+  /** */
+  clear() {
+    if (this.context2d) {
+      this.context2d.clearRect(0, 0, this.containerWidth, this.containerHeight);
+    }
+  }
+
+  /**
+   * resize - resizes the added Canvas overlay.
+   */
+  resize() {
+    if (!this.video || !this.canvas) { return; }
+
+    if (this.containerWidth !== this.video.clientWidth) {
+      this.containerWidth = this.video.clientWidth;
+      this.canvas.setAttribute('width', this.containerWidth);
+    }
+
+    if (this.containerHeight !== this.video.clientHeight) {
+      this.containerHeight = this.video.clientHeight;
+      this.canvas.setAttribute('height', this.containerHeight);
+    }
+  }
+
+  /**
+   * canvasUpdate
+   * @param {Function} update
+   */
+  canvasUpdate(update) {
+    if (!this.context2d) { return; }
+
+    const ratio = this.scale;
+    this.context2d.scale(ratio, ratio);
+    update();
+    this.context2d.setTransform(1, 0, 0, 1, 0, 0);
+  }
+}

src/state/actions/action-types.js

@@ -72,6 +72,10 @@ const ActionTypes = {
   REMOVE_RESOURCE: 'mirador/REMOVE_RESOURCE',
   SHOW_COLLECTION_DIALOG: 'mirador/SHOW_COLLECTION_DIALOG',
   HIDE_COLLECTION_DIALOG: 'mirador/HIDE_COLLECTION_DIALOG',
+
+  SET_CURRENT_TIME: 'mirador/SET_CURRENT_TIME',
+  SET_VIDEO_PAUSED: 'mirador/SET_VIDEO_PAUSED',
+  SET_VIDEO_MUTED: 'mirador/SET_VIDEO_MUTED',
 };
 
 export default ActionTypes;

src/state/actions/window.js

@@ -204,3 +204,36 @@ export function hideCollectionDialog(windowId) {
     windowId,
   };
 }
+
+/** */
+export function setWindowCurrentTime(windowId, currentTime) {
+  return ((dispatch) => {
+    dispatch({
+      currentTime,
+      type: ActionTypes.SET_CURRENT_TIME,
+      windowId,
+    });
+  });
+}
+
+/** */
+export function setWindowPaused(windowId, paused) {
+  return ((dispatch) => {
+    dispatch({
+      paused: (paused === undefined) ? true : paused,
+      type: ActionTypes.SET_VIDEO_PAUSED,
+      windowId,
+    });
+  });
+}
+
+/** */
+export function setWindowMuted(windowId, muted) {
+  return ((dispatch) => {
+    dispatch({
+      muted: (muted === undefined) ? false : muted,
+      type: ActionTypes.SET_VIDEO_MUTED,
+      windowId,
+    });
+  });
+}

src/state/reducers/windows.js

@@ -75,6 +75,7 @@ export const windowsReducer = (state = {}, action) => {
         {
           ...(orig || {}),
           canvasId: action.canvasId,
+          currentTime: 0,
           visibleCanvases: action.visibleCanvases || [],
         }), state);
     case ActionTypes.ADD_COMPANION_WINDOW:
@@ -168,6 +169,30 @@ export const windowsReducer = (state = {}, action) => {
           collectionDialogOn: false,
         },
       };
+    case ActionTypes.SET_CURRENT_TIME:
+      return {
+        ...state,
+        [action.windowId]: {
+          ...state[action.windowId],
+          currentTime: action.currentTime,
+        },
+      };
+    case ActionTypes.SET_VIDEO_PAUSED:
+      return {
+        ...state,
+        [action.windowId]: {
+          ...state[action.windowId],
+          paused: !!action.paused,
+        },
+      };
+    case ActionTypes.SET_VIDEO_MUTED:
+      return {
+        ...state,
+        [action.windowId]: {
+          ...state[action.windowId],
+          muted: !!action.muted,
+        },
+      };
     default:
       return state;
   }

src/state/selectors/canvases.js

@@ -229,3 +229,15 @@ export const selectInfoResponse = createSelector(
     && infoResponses[iiifServiceId];
   },
 );
+
+export const getCurrentCanvasDuration = createSelector(
+  [
+    getCurrentCanvas,
+  ],
+  (canvas) => {
+    if (canvas && canvas.__jsonld && 'duration' in canvas.__jsonld) {
+      return canvas.__jsonld.duration;
+    }
+    return undefined;
+  },
+);

src/state/selectors/index.js

@@ -13,3 +13,4 @@ export * from './sequences';
 export * from './auth';
 export * from './utils';
 export * from './viewer';
+export * from './window';