Select Git revision
miradorAnnotationPlugin.test.js
AnnotationsOverlayVideo.js 22.12 KiB
import React, { Component } from 'react';
import PropTypes from 'prop-types';
import isEqual from 'lodash/isEqual';
import debounce from 'lodash/debounce';
import flatten from 'lodash/flatten';
import sortBy from 'lodash/sortBy';
import xor from 'lodash/xor';
import ResizeObserver from 'react-resize-observer';
import CircularProgress from '@material-ui/core/CircularProgress';
import CanvasOverlayVideo from '../lib/CanvasOverlayVideo';
import CanvasWorld from '../lib/CanvasWorld';
import CanvasAnnotationDisplay from '../lib/CanvasAnnotationDisplay';
import { VideosReferences } from '../plugins/VideosReferences';
/** AnnotationsOverlayVideo - based on AnnotationsOverlay */
export class AnnotationsOverlayVideo extends Component {
/**
* annotationsMatch - compares previous annotations to current to determine
* whether to add a new updateCanvas method to draw annotations
* @param {Array} currentAnnotations
* @param {Array} prevAnnotations
* @return {Boolean}
*/
static annotationsMatch(currentAnnotations, prevAnnotations) {
if (!currentAnnotations && !prevAnnotations) return true;
if (
(currentAnnotations && !prevAnnotations)
|| (!currentAnnotations && prevAnnotations)
) return false;
if (currentAnnotations.length === 0 && prevAnnotations.length === 0) return true;
if (currentAnnotations.length !== prevAnnotations.length) return false;
return currentAnnotations.every((annotation, index) => {
const newIds = annotation.resources.map(r => r.id);
const prevIds = prevAnnotations[index].resources.map(r => r.id);
if (newIds.length === 0 && prevIds.length === 0) return true;
if (newIds.length !== prevIds.length) return false;
if ((annotation.id === prevAnnotations[index].id) && (isEqual(newIds, prevIds))) {
return true;
}
return false;
});
}
/** @private */
static isAnnotaionInTemporalSegment(resource, time) {
const temporalfragment = resource.temporalfragmentSelector;
if (temporalfragment && temporalfragment.length > 0) {
const start = temporalfragment[0] || 0;
const end = (temporalfragment.length > 1) ? temporalfragment[1] : Number.MAX_VALUE;
if (start <= time && time < end) {
//
} else {
return false;
}
}
return true;
}
/**
* @param {Object} props
*/
constructor(props) {
super(props);
this.ref = React.createRef();
VideosReferences.set(props.windowId, this);
this.canvasOverlay = null;
// An initial value for the updateCanvas method this.updateCanvas = () => {};
this.onCanvasClick = this.onCanvasClick.bind(this);
this.onCanvasMouseMove = debounce(this.onCanvasMouseMove.bind(this), 10);
this.onCanvasExit = this.onCanvasExit.bind(this);
this.onVideoTimeUpdate = this.onVideoTimeUpdate.bind(this);
this.onVideoLoadedMetadata = this.onVideoLoadedMetadata.bind(this);
this.onVideoWaiting = this.onVideoWaiting.bind(this);
this.onVideoPlaying = this.onVideoPlaying.bind(this);
this.onCanvasResize = this.onCanvasResize.bind(this);
this.imagesLoading = [];
this.imagesReady = [];
const { videoTarget: temporalfragment } = this.props;
if (temporalfragment && temporalfragment.length > 0) {
this.temporalOffset = temporalfragment[0] || 0;
} else {
this.temporalOffset = 0;
}
this.currentTimeNearestAnnotationId = null;
this.state = {
showProgress: false,
};
}
/**
* React lifecycle event
*/
componentDidMount() {
this.initializeViewer();
}
/** */
componentDidUpdate(prevProps) {
const {
canvasWorld,
currentTime,
drawAnnotations,
drawSearchAnnotations,
annotations, searchAnnotations,
hoveredAnnotationIds, selectedAnnotationId,
highlightAllAnnotations,
paused,
seekToTime,
} = this.props;
this.initializeViewer();
let prevVideoPausedState;
if (this.video) {
prevVideoPausedState = this.video.paused;
if (this.video.paused && !paused) {
const promise = this.video.play();
if (promise !== undefined) {
promise.catch((e) => {});
}
} else if (!this.video.paused && paused) {
this.video.pause();
}
if (seekToTime !== prevProps.seekToTime) {
if (seekToTime !== undefined) {
this.seekTo(seekToTime, true);
return;
}
}
if (this.video.seeking) {
return; }
if (currentTime !== prevProps.currentTime) {
if (paused && this.video.paused) {
this.video.currentTime = currentTime - this.temporalOffset;
}
}
}
const annotationsUpdated = !AnnotationsOverlayVideo.annotationsMatch(
annotations, prevProps.annotations,
);
const searchAnnotationsUpdated = !AnnotationsOverlayVideo.annotationsMatch(
searchAnnotations, prevProps.searchAnnotations,
);
const hoveredAnnotationsUpdated = (
xor(hoveredAnnotationIds, prevProps.hoveredAnnotationIds).length > 0
);
if (this.canvasOverlay && this.canvasOverlay.canvas && hoveredAnnotationsUpdated) {
if (hoveredAnnotationIds.length > 0) {
this.canvasOverlay.canvas.style.cursor = 'pointer';
} else {
this.canvasOverlay.canvas.style.cursor = '';
}
}
const selectedAnnotationsUpdated = selectedAnnotationId !== prevProps.selectedAnnotationId;
if (selectedAnnotationsUpdated && selectedAnnotationId) {
if (this.currentTimeNearestAnnotationId
&& this.currentTimeNearestAnnotationId === selectedAnnotationId) {
// go through
} else {
annotations.forEach((annotation) => {
annotation.resources.forEach((resource) => {
if (resource.id !== selectedAnnotationId) return;
if (!canvasWorld.canvasIds.includes(resource.targetId)) return;
if (!AnnotationsOverlayVideo.isAnnotaionInTemporalSegment(resource, currentTime)) {
const temporalfragment = resource.temporalfragmentSelector;
if (temporalfragment && temporalfragment.length > 0 && this.video) {
const seekto = temporalfragment[0] || 0;
this.seekTo(seekto, !prevVideoPausedState);
}
}
});
});
}
}
// auto scroll
if (this.video && !this.video.paused) {
let minElapsedTimeAfterStart = Number.MAX_VALUE;
let candidateAnnotation;
annotations.forEach((annotation) => {
annotation.resources.forEach((resource) => {
if (!canvasWorld.canvasIds.includes(resource.targetId)) return;
if (AnnotationsOverlayVideo.isAnnotaionInTemporalSegment(resource, currentTime)) {
const temporalfragment = resource.temporalfragmentSelector;
if (temporalfragment && temporalfragment.length > 0 && this.video) {
const seekto = temporalfragment[0] || 0;
const elapsedTimeAfterStart = currentTime - seekto;
if (elapsedTimeAfterStart >= 0 && elapsedTimeAfterStart < minElapsedTimeAfterStart) {
minElapsedTimeAfterStart = elapsedTimeAfterStart;
candidateAnnotation = resource.resource;
}
}
}
});
});
if (candidateAnnotation) { if (candidateAnnotation.id !== prevProps.selectedAnnotationId) {
const {
selectAnnotation,
windowId,
} = this.props;
if (selectedAnnotationId !== candidateAnnotation.id) {
selectAnnotation(windowId, candidateAnnotation.id);
}
this.currentTimeNearestAnnotationId = candidateAnnotation.id;
}
}
}
const redrawAnnotations = drawAnnotations !== prevProps.drawAnnotations
|| drawSearchAnnotations !== prevProps.drawSearchAnnotations
|| highlightAllAnnotations !== prevProps.highlightAllAnnotations;
if (
searchAnnotationsUpdated
|| annotationsUpdated
|| selectedAnnotationsUpdated
|| hoveredAnnotationsUpdated
|| redrawAnnotations
) {
this.updateCanvas = this.canvasUpdateCallback();
this.updateCanvas();
}
}
/**
*/
componentWillUnmount() {
if (this.resizeObserver) {
this.resizeObserver.disconnect();
}
if (this.video) {
this.video.removeEventListener('timeupdate', this.onVideoTimeUpdate);
this.video.removeEventListener('loadedmetadata', this.onVideoLoadedMetadata);
this.video.removeEventListener('waiting', this.onVideoWaiting);
this.video.removeEventListener('playing', this.onVideoPlaying);
this.video.removeEventListener('seeked', this.onVideoPlaying);
}
if (this.canvasOverlay && this.canvasOverlay.canvas) {
this.canvasOverlay.canvas.removeEventListener('click', this.onCanvasClick);
this.canvasOverlay.canvas.removeEventListener('mouseleave', this.onCanvasExit);
this.canvasOverlay.canvas.removeEventListener('mousemove', this.onCanvasMouseMove);
}
}
/** */
onVideoTimeUpdate(event) {
this.updateCanvas();
}
/** */
onVideoLoadedMetadata(event) {
if (this.video) {
const { currentTime } = this.props;
const { temporalOffset } = this;
this.video.currentTime = currentTime - temporalOffset;
}
}
/** */
onVideoPlaying(event) {
if (this.video && this.video.currentTime !== 0) {
const { currentTime, seekToTime } = this.props;
const currentTimeToVideoTime = currentTime - this.temporalOffset;
const diff = Math.abs(currentTimeToVideoTime - this.video.currentTime);
const acceptableDiff = 1; // sec. if (diff > acceptableDiff && seekToTime === undefined) {
this.seekTo(this.video.currentTime + this.temporalOffset, true);
}
}
this.setState({ showProgress: false });
}
/** */
onVideoWaiting(event) {
this.setState({ showProgress: true });
}
/** */
onCanvasClick(event) {
const { canvas: canvas_, canvasWorld, currentTime } = this.props;
const scale = (this.canvasOverlay ? this.canvasOverlay.scale : 1) || 1;
const point = { x: event.layerX / scale, y: event.layerY / scale };
const canvas = this.isCanvasSizeSpecified() ? canvasWorld.canvasAtPoint(point) : canvas_;
if (!canvas) return;
// get all the annotations that contain the click
// const currentTime = this.video ? this.video.currentTime : undefined;
const annos = this.annotationsAtPoint(canvas, point, currentTime);
if (annos.length > 0) {
event.preventDefaultAction = true; // eslint-disable-line no-param-reassign
}
if (annos.length === 1) {
this.toggleAnnotation(annos[0].id);
} else if (annos.length > 0) {
/**
* Try to find the "right" annotation to select after a click.
*
* This is perhaps a naive method, but seems to deal with rectangles and SVG shapes:
*
* - figure out how many points around a circle are inside the annotation shape
* - if there's a shape with the fewest interior points, it's probably the one
* with the closest boundary?
* - if there's a tie, make the circle bigger and try again.
*/
const annosWithClickScore = (radius) => {
const degreesToRadians = Math.PI / 180;
return (anno) => {
let score = 0;
for (let degrees = 0; degrees < 360; degrees += 1) {
const x = Math.cos(degrees * degreesToRadians) * radius + point.x;
const y = Math.sin(degrees * degreesToRadians) * radius + point.y;
if (this.isAnnotationAtPoint(anno, canvas, { x, y })) score += 1;
}
return { anno, score };
};
};
let annosWithScore = [];
let radius = 1;
annosWithScore = sortBy(annos.map(annosWithClickScore(radius)), 'score');
const { width: canvasWidth, height: canvasHeight } = this.getCurrentCanvasSize();
while (radius < Math.max(canvasWidth, canvasHeight)
&& annosWithScore[0].score === annosWithScore[1].score) {
radius *= 2;
annosWithScore = sortBy(annos.map(annosWithClickScore(radius)), 'score');
}
this.toggleAnnotation(annosWithScore[0].anno.id);
}
}
/** */
onCanvasMouseMove(event) {
const {
annotations,
canvas: canvas_,
canvasWorld,
currentTime,
hoverAnnotation,
hoveredAnnotationIds,
searchAnnotations,
windowId,
} = this.props;
if (annotations.length === 0 && searchAnnotations.length === 0) return;
const scale = (this.canvasOverlay ? this.canvasOverlay.scale : 1) || 1;
const point = { x: event.layerX / scale, y: event.layerY / scale };
const canvas = this.isCanvasSizeSpecified() ? canvasWorld.canvasAtPoint(point) : canvas_;
if (!canvas) {
hoverAnnotation(windowId, []);
return;
}
// const currentTime = this.video ? this.video.currentTime : undefined;
const annos = this.annotationsAtPoint(canvas, point, currentTime);
if (xor(hoveredAnnotationIds, annos.map(a => a.id)).length > 0) {
hoverAnnotation(windowId, annos.map(a => a.id));
}
}
/** If the cursor leaves the canvas, wipe out highlights */
onCanvasExit(event) {
const { hoverAnnotation, windowId } = this.props;
// a move event may be queued up by the debouncer
this.onCanvasMouseMove.cancel();
hoverAnnotation(windowId, []);
}
/**
* Make sure that the annotation position on the Canvas is correct even when the video
* display size is changed by opening and closing the side panel.
*/
onCanvasResize(event) {
this.updateCanvas();
}
/** @private */
getCurrentCanvasSize() {
const { canvas, canvasWorld } = this.props;
const [
_canvasX, _canvasY, _canvasWidth, _canvasHeight, // eslint-disable-line no-unused-vars
] = canvasWorld.canvasToWorldCoordinates(canvas.id);
if (_canvasWidth && _canvasHeight) {
return { height: _canvasHeight, width: _canvasWidth };
}
if (this.video) {
const { videoWidth, videoHeight } = this.video;
return { height: videoHeight, width: videoWidth };
}
return { height: 0, width: 0 };
}
/** @private */ getResourceImage(resource) {
let imageSource;
if (resource.body && resource.body.length > 0 && resource.body[0].type === 'Image') {
const src = resource.body[0].id;
if (this.imagesReady[src]) {
imageSource = this.imagesReady[src];
} else if (!this.imagesLoading.includes(src)) {
this.imagesLoading.push(src);
const img = new Image();
img.addEventListener('load', () => {
this.imagesReady[src] = img;
}, false);
img.src = src;
}
}
return imageSource;
}
/** @private */
seekTo(seekTo, resume) {
const { setCurrentTime, setPaused } = this.props;
setPaused(true);
setCurrentTime(seekTo);
this.video.addEventListener('seeked', function seeked(event) {
event.currentTarget.removeEventListener(event.type, seeked);
if (resume) {
setPaused(false);
}
});
}
/** @private */
isCanvasSizeSpecified() {
const { canvas, canvasWorld } = this.props;
const [
_canvasX, _canvasY, _canvasWidth, _canvasHeight, // eslint-disable-line no-unused-vars
] = canvasWorld.canvasToWorldCoordinates(canvas.id);
return _canvasWidth && _canvasHeight;
}
/** @private */
initializeViewer() {
if (this.canvasOverlay && this.canvasOverlay.canvas) {
this.canvasOverlay.canvas.addEventListener('click', this.onCanvasClick);
this.canvasOverlay.canvas.addEventListener('mouseleave', this.onCanvasExit);
this.canvasOverlay.canvas.addEventListener('mousemove', this.onCanvasMouseMove);
}
if (this.canvasOverlay) return;
const { videoRef } = this.props;
if (!videoRef.current) return;
this.video = videoRef.current;
this.video.addEventListener('timeupdate', this.onVideoTimeUpdate);
this.video.addEventListener('loadedmetadata', this.onVideoLoadedMetadata);
this.video.addEventListener('waiting', this.onVideoWaiting);
this.video.addEventListener('playing', this.onVideoPlaying);
this.video.addEventListener('seeked', this.onVideoPlaying);
const { canvas, canvasWorld } = this.props;
const canvasSize = canvasWorld.canvasToWorldCoordinates(canvas.id);
this.canvasOverlay = new CanvasOverlayVideo(this.video, this.ref, canvasSize);
this.updateCanvas = this.canvasUpdateCallback();
// Prefetch annotation images
const { annotations } = this.props;
annotations.forEach((annotation) => {
annotation.resources.forEach((resource) => {
if (!canvasWorld.canvasIds.includes(resource.targetId)) return;
this.getResourceImage(resource); });
});
}
/** */
canvasUpdateCallback() {
return () => {
this.canvasOverlay.clear();
this.canvasOverlay.resize();
this.canvasOverlay.canvasUpdate(this.renderAnnotations.bind(this));
};
}
/** @private */
isAnnotationAtPoint(resource, canvas, point, time) {
const { canvasWorld } = this.props;
const [canvasX, canvasY] = canvasWorld.canvasToWorldCoordinates(canvas.id);
const relativeX = point.x - canvasX;
const relativeY = point.y - canvasY;
if (resource.temporalfragmentSelector) {
if (!AnnotationsOverlayVideo.isAnnotaionInTemporalSegment(resource, time)) {
return false;
}
}
if (resource.svgSelector) {
const context = this.canvasOverlay.context2d;
const { svgPaths } = new CanvasAnnotationDisplay({ resource });
return [...svgPaths].some(path => (
context.isPointInPath(new Path2D(path.attributes.d.nodeValue), relativeX, relativeY)
));
}
if (resource.fragmentSelector) {
const [x, y, w, h] = resource.fragmentSelector;
return (x <= relativeX && relativeX <= (x + w) && y <= relativeY && relativeY <= (y + h));
}
// If there is no svgSelector or fragmentSelector, assume that the target is the entire canvas.
return true;
}
/** @private */
annotationsAtPoint(canvas, point, time) {
const { annotations, searchAnnotations } = this.props;
const lists = [...annotations, ...searchAnnotations];
const annos = flatten(lists.map(l => l.resources)).filter((resource) => {
if (canvas.id !== resource.targetId) return false;
return this.isAnnotationAtPoint(resource, canvas, point, time);
});
return annos;
}
/** */
toggleAnnotation(id) {
const {
selectedAnnotationId,
selectAnnotation,
deselectAnnotation,
windowId,
} = this.props;
if (selectedAnnotationId === id) {
deselectAnnotation(windowId, id);
} else { selectAnnotation(windowId, id);
}
}
/**
* annotationsToContext - converts anontations to a canvas context
*/
annotationsToContext(annotations, palette) {
const {
highlightAllAnnotations, hoveredAnnotationIds, selectedAnnotationId, canvasWorld, currentTime,
} = this.props;
const context = this.canvasOverlay.context2d;
const zoomRatio = 1;
annotations.forEach((annotation) => {
annotation.resources.forEach((resource) => {
if (!canvasWorld.canvasIds.includes(resource.targetId)) return;
if (!AnnotationsOverlayVideo.isAnnotaionInTemporalSegment(resource, currentTime)) return;
const imageSource = this.getResourceImage(resource);
const offset = canvasWorld.offsetByCanvas(resource.targetId);
const canvasSize = canvasWorld.canvasToWorldCoordinates(resource.targetId);
const canvasAnnotationDisplay = new CanvasAnnotationDisplay({
canvasSize,
hovered: hoveredAnnotationIds.includes(resource.id),
imageSource,
offset,
palette: {
...palette,
default: {
...palette.default,
...(!highlightAllAnnotations && palette.hidden),
},
},
resource,
selected: selectedAnnotationId === resource.id,
zoomRatio,
});
canvasAnnotationDisplay.toContext(context);
});
});
}
/** */
renderAnnotations() {
const {
annotations,
drawAnnotations,
drawSearchAnnotations,
searchAnnotations,
palette,
} = this.props;
if (drawSearchAnnotations) {
this.annotationsToContext(searchAnnotations, palette.search);
}
if (drawAnnotations) {
this.annotationsToContext(annotations, palette.annotations);
}
}
/**
* Renders things
*/
render() {
const { showProgress } = this.state;
const circularProgress = (<CircularProgress style={{ left: '50%', position: 'absolute', top: '50%' }} />);
return (
<>
<canvas ref={this.ref} style={{ left: 0, position: 'absolute', top: 0 }} /> <ResizeObserver onResize={this.onCanvasResize} />
{ showProgress && circularProgress }
</>
);
}
}
AnnotationsOverlayVideo.defaultProps = {
annotations: [],
canvas: {},
currentTime: 0,
deselectAnnotation: () => {},
drawAnnotations: true,
drawSearchAnnotations: true,
highlightAllAnnotations: false,
hoverAnnotation: () => {},
hoveredAnnotationIds: [],
palette: {},
paused: true,
searchAnnotations: [],
seekToTime: undefined,
selectAnnotation: () => {},
selectedAnnotationId: undefined,
setCurrentTime: () => {},
setPaused: () => {},
videoTarget: [],
};
AnnotationsOverlayVideo.propTypes = {
annotations: PropTypes.arrayOf(PropTypes.object),
canvas: PropTypes.object, // eslint-disable-line react/forbid-prop-types
canvasWorld: PropTypes.instanceOf(CanvasWorld).isRequired,
currentTime: PropTypes.number,
deselectAnnotation: PropTypes.func,
drawAnnotations: PropTypes.bool,
drawSearchAnnotations: PropTypes.bool,
highlightAllAnnotations: PropTypes.bool,
hoverAnnotation: PropTypes.func,
hoveredAnnotationIds: PropTypes.arrayOf(PropTypes.string),
palette: PropTypes.object, // eslint-disable-line react/forbid-prop-types
paused: PropTypes.bool,
searchAnnotations: PropTypes.arrayOf(PropTypes.object),
seekToTime: PropTypes.number,
selectAnnotation: PropTypes.func,
selectedAnnotationId: PropTypes.string,
setCurrentTime: PropTypes.func,
setPaused: PropTypes.func,
videoRef: PropTypes.object.isRequired, // eslint-disable-line react/forbid-prop-types
videoTarget: PropTypes.arrayOf(PropTypes.number),
windowId: PropTypes.string.isRequired,
};