Merge pull request #136 from spacedeck/fix-video-audio-processing

Fix video/audio conversion real-time progress and finalizing

helpers/artifact_converter.js

@@ -82,7 +82,7 @@ function createWaveform(fileName, localFilePath, callback){
         "-i", localFilePath, "-o", filePathImage
       ],
     {}, function(error, stdout, stderr) {
-      if(!error) {
+      if (!error) {
         callback(null, filePathImage);
       } else {
         console.log("error:", stdout, stderr);
@@ -99,7 +99,7 @@ function convertVideo(fileName, filePath, codec, callback, progressCallback) {
   var newExt = codec == "mp4" ? "mp4" : "ogv";
   var convertedPath = filePath + "." + newExt;
 
-  console.log("converting", filePath, "to", convertedPath);
+  console.log("convertVideo", filePath, "to", convertedPath);
 
   var convertArgs = (codec == "mp4") ? [
     "-i", filePath,
@@ -141,7 +141,7 @@ function convertVideo(fileName, filePath, codec, callback, progressCallback) {
 
   ff.on('close', function (code) {
     console.log('[ffmpeg-video] child process exited with code ' + code);
-    if (!code) { 
+    if (!code) {
       console.log("converted", filePath, "to", convertedPath);
       callback(null, convertedPath);
     } else {
@@ -190,7 +190,7 @@ function createThumbnailForVideo(fileName, filePath, callback) {
 function getMime(fileName, filePath, callback) {
   var ext = path.extname(fileName);
   var presetMime = mime.lookup(fileName);
-  
+
   if (presetMime) {
     callback(null, presetMime);
   } else {
@@ -229,7 +229,6 @@ function resizeAndUpload(a, size, max, fileName, localFilePath, callback) {
   }
 }
 
-
 var resizeAndUploadImage = function(a, mimeType, size, fileName, fileNameOrg, imageFilePath, originalFilePath, payloadCallback) {
   async.parallel({
     small: function(callback){
@@ -281,17 +280,17 @@ var resizeAndUploadImage = function(a, mimeType, size, fileName, fileNameOrg, im
 
 module.exports = {
   convert: function(a, fileName, localFilePath, payloadCallback, progressCallback) {
-    getMime(fileName, localFilePath, function(err, mimeType){
+    getMime(fileName, localFilePath, function(err, mimeType) {
       console.log("[convert] fn: "+fileName+" local: "+localFilePath+" mimeType:", mimeType);
 
       if (!err) {
         if (convertableImageTypes.indexOf(mimeType) > -1) {
-         
+
           gm(localFilePath).size(function (err, size) {
             console.log("[convert] gm:", err, size);
 
             if (!err) {
-              if(mimeType == "application/pdf") {
+              if (mimeType == "application/pdf") {
                 var firstImagePath =  localFilePath + ".jpeg";
                 exec.execFile("gs", ["-sDEVICE=jpeg","-dNOPAUSE", "-dJPEGQ=80", "-dBATCH", "-dFirstPage=1", "-dLastPage=1", "-sOutputFile=" + firstImagePath, "-r90", "-f", localFilePath], {}, function(error, stdout, stderr) {
                   if(error === null) {
@@ -305,7 +304,7 @@ module.exports = {
                   }
                 });
 
-              } else if(mimeType == "image/gif") {
+              } else if (mimeType == "image/gif") {
                 //gifs are buggy after convertion, so we should not convert them
 
                 var s3Key = "s"+ a.space_id.toString() + "/a" + a._id.toString() + "/" + fileName;
@@ -334,7 +333,7 @@ module.exports = {
 
                     a.save().then(function() {
                       fs.unlink(localFilePath, function (err) {
-                        if (err){
+                        if (err) {
                           console.error(err);
                           payloadCallback(err, null);
                         } else {
@@ -350,8 +349,8 @@ module.exports = {
                 resizeAndUploadImage(a, mimeType, size, fileName, fileName, localFilePath, localFilePath, payloadCallback);
               }
             } else payloadCallback(err);
-          });            
+          });
-      
+
         } else if (convertableVideoTypes.indexOf(mimeType) > -1) {
           async.parallel({
             thumbnail: function(callback) {
@@ -453,11 +452,10 @@ module.exports = {
           });
 
         } else if (convertableAudioTypes.indexOf(mimeType) > -1) {
-
           async.parallel({
             ogg: function(callback) {
               convertAudio(fileName, localFilePath, "ogg", function(err, file) {
-                if(err) callback(err);
+                if (err) callback(err);
                 else {
                   var keyName = "s" + a.space_id.toString() + "/a" + a._id.toString() + "/" + fileName + ".ogg" ;
                   uploader.uploadFile(keyName, "audio/ogg", file, function(err, url){
@@ -469,20 +467,16 @@ module.exports = {
             },
             mp3_waveform: function(callback) {
               convertAudio(fileName, localFilePath, "mp3", function(err, file) {
-                if(err) callback(err);
+                if (err) callback(err);
                 else {
-
+                  createWaveform(fileName, file, function(err, filePath) {
-                  createWaveform(fileName, file, function(err, filePath){
-
                     var keyName = "s" + a.space_id.toString() + "/a" + a._id.toString() + "/" + fileName + "-" + (new Date().getTime()) + ".png";
-                    uploader.uploadFile(keyName, "image/png", filePath, function(err, pngUrl){
+                    uploader.uploadFile(keyName, "image/png", filePath, function(err, pngUrl) {
-
                       var keyName = "s" + a.space_id.toString() + "/a" + a._id.toString() + "/" + fileName + ".mp3" ;
-                      uploader.uploadFile(keyName, "audio/mp3", file, function(err, mp3Url){
+                      uploader.uploadFile(keyName, "audio/mp3", file, function(err, mp3Url) {
                         if (err) callback(err);
                         else callback(null, {waveform: pngUrl, mp3: mp3Url});
                       });
-
                     });
                   });
                 }
@@ -490,7 +484,7 @@ module.exports = {
             },
             original: function(callback) {
               var keyName = "s" + a.space_id.toString() + "/a" + a._id.toString() + "/" + fileName;
-              uploader.uploadFile(keyName, mimeType, localFilePath, function(err, url){
+              uploader.uploadFile(keyName, mimeType, localFilePath, function(err, url) {
                 callback(null, url);
               });
             }
@@ -499,7 +493,6 @@ module.exports = {
 
             if (err) payloadCallback(err, a);
             else {
-
               a.state = "idle";
               a.mime = mimeType;
               var stats = fs.statSync(localFilePath);
@@ -515,10 +508,9 @@ module.exports = {
               ];
 
               a.updated_at = new Date();
-
               db.packArtifact(a);
-              
+
-              a.save().then(function(){
+              a.save().then(function() {
                 fs.unlink(localFilePath, function (err) {
                   if (err){
                     console.error(err);
@@ -537,13 +529,13 @@ module.exports = {
           console.log("mimeType not matched for conversion, storing file");
           var keyName = "s" + a.space_id.toString() + "/a" + a._id.toString() + "/" + fileName;
           uploader.uploadFile(keyName, mimeType, localFilePath, function(err, url) {
-            
+
             a.state = "idle";
             a.mime = mimeType;
             var stats = fs.statSync(localFilePath);
             a.payload_size = stats["size"];
             a.payload_uri = url;
-            
+
             a.updated_at = new Date();
             a.save().then(function() {
               fs.unlink(localFilePath, function (err) {
@@ -559,5 +551,3 @@ module.exports = {
     });
   }
 };
-
-

middlewares/api_helpers.js

@@ -17,9 +17,15 @@ module.exports = (req, res, next) => {
     this.status(201).json(object);
   };
 
-  res['distributeUpdate'] = function(model, object) {
+  res['distributeUpdate'] = function(model, object, sendToSelf) {
     if (!object) return;
-    redis.sendMessage("update", model, object, req.channelId);
+    if (sendToSelf) {
+      // send this update to the initiating user, for example when
+      // a conversion task has finished
+      redis.sendMessage("update-self", model, object, req.channelId);
+    } else {
+      redis.sendMessage("update", model, object, req.channelId);
+    }
     this.status(200).json(object);
   };
 

public/javascripts/spacedeck_sections.js

@@ -1602,12 +1602,8 @@ var SpacedeckSections = {
       if (this.guest_nickname) {
         new_item.editor_name = this.guest_nickname;
       }
-      // console.log("new artifact", new_item);
 
       save_artifact(new_item, function(saved_item) {
-
-        // console.log("saved artifact", saved_item);
-
         this.update_board_artifact_viewmodel(saved_item);
         this.active_space_artifacts.push(saved_item);
 

public/javascripts/spacedeck_websockets.js

@@ -40,7 +40,11 @@ SpacedeckWebsockets = {
             }
           } else console.log("artifact created in another space.");
         }
-        else if (msg.action == "update" && msg.object) {
+        else if ((msg.action == "update" || msg.action == "update-self") && msg.object) {
+          if (msg.action == "update-self") {
+            console.log(msg.object);
+          }
+
           if (this.active_space) {
             var o = msg.object;
             if (o && o._id) {
@@ -185,7 +189,7 @@ SpacedeckWebsockets = {
           return;
         }
 
-        if (msg.channel_id == channel_id) {
+        if (msg.channel_id == channel_id && !msg.action.match("-self")) {
           return;
         }
 
@@ -199,7 +203,7 @@ SpacedeckWebsockets = {
           this.handle_presenter_media_update(msg);
         }
 
-        if (msg.action == "update" || msg.action == "create" || msg.action == "delete"){
+        if (msg.action == "update" || msg.action == "update-self" || msg.action == "create" || msg.action == "delete") {
           this.handle_live_updates(msg);
         }
 
@@ -232,13 +236,13 @@ SpacedeckWebsockets = {
               return (u && (u._id != this.user._id));
             }.bind(this));
           }
-          
+
           users = _.filter(users, function(u) {
             return (u && (u._id || u.nickname));
           });
-          
+
           this.users_online[spaceId] = users;
-          
+
           if (this.active_space) {
             if (this.active_space._id == spaceId) {
               this.active_space_users = users;

public/stylesheets/style.css

@@ -15350,7 +15350,8 @@ body:not(.present-mode) #space .artifact.selected {
     padding: 10px;
     background-color: #3d9ee9;
     opacity: 0.9;
-    text-align: center; }
+    text-align: center;
+    font-size: 14px; }
   .artifact.state-processing .progress-text, .artifact.state-uploading .progress-text {
     text-align: center;
     padding: 8px;

routes/api/space_artifacts.js

@@ -89,7 +89,7 @@ router.post('/', function(req, res, next) {
 
   var artifact = attrs;
   artifact._id = uuidv4();
-  
+
   if (req.user) {
     artifact.user_id = req.user._id;
     artifact.last_update_user_id = req.user._id;
@@ -114,7 +114,7 @@ router.post('/', function(req, res, next) {
 });
 
 router.post('/:artifact_id/payload', function(req, res, next) {
-  if (req.spaceRole == "editor"  ||  req.spaceRole == "admin") {
+  if (req.spaceRole == "editor" || req.spaceRole == "admin") {
     var a = req.artifact;
 
     var fileName = (req.query.filename || "upload.bin").replace(/[^a-zA-Z0-9_\-\.]/g, '');
@@ -124,10 +124,21 @@ router.post('/:artifact_id/payload', function(req, res, next) {
     var stream = req.pipe(writeStream);
 
     var progressCallback = function(progressMsg) {
-      a.description = progressMsg.toString();
+      // merge progress message with any other changes (size/location)
-      db.packArtifact(a);
+      db.Artifact.findOne({where: {
-      a.save();
+        _id: a._id
-      redis.sendMessage("update", "Artifact", a, req.channelId);
+      }}).then(a => {
+        if (a) {
+          a.description = progressMsg.toString();
+          db.packArtifact(a);
+          a.save();
+          db.unpackArtifact(a);
+          redis.sendMessage("update-self", "Artifact", a, req.channelId);
+        } else {
+          // artifact has been deleted
+          // TODO: stop conversion process!
+        }
+      });
     };
 
     stream.on('finish', function() {
@@ -135,7 +146,8 @@ router.post('/:artifact_id/payload', function(req, res, next) {
         if (error) res.status(400).json(error);
         else {
           db.Space.update({ updated_at: new Date() }, {where: {_id: req.space._id}});
-          res.distributeUpdate("Artifact", artifact);
+          db.unpackArtifact(artifact);
+          res.distributeUpdate("Artifact", artifact, true);
         }
       }, progressCallback);
     });
@@ -157,7 +169,7 @@ router.put('/:artifact_id', function(req, res, next) {
   } else {
     newAttr.last_update_editor_name = req.editor_name;
   }
-  
+
   db.packArtifact(newAttr);
 
   db.Artifact.update(newAttr, { where: {

styles/artifact.scss

@@ -288,7 +288,7 @@
         pointer-events: none;
       }
     }
-    
+
     .title {font-size: 20px; }
     .image {
       height: auto;
@@ -335,7 +335,7 @@
       padding: 10px 15px;
       font-size: 10px;
     }
-    
+
     video {
       width: 100%;
       height: 100%;
@@ -501,7 +501,7 @@ body:not(.present-mode) {
     .Medium {
       cursor: text;
     }
-    
+
     .artifact {
 
       &.selected.text-editing,
@@ -566,16 +566,17 @@ body:not(.present-mode) {
   .progress {
     height: 100%;
     padding: 10px;
-    
+
     background-color: $blue;
     opacity: 0.9;
     text-align: center;
+    font-size: 14px;
   }
   .progress-text {
     text-align: center;
     padding: 8px;
     width: 100%;
-    
+
     position: absolute;
     left: 0px;
     top: 0px;
@@ -586,8 +587,8 @@ body:not(.present-mode) {
   video, audio, .tl-controls, .timeline {
     display: none;
   }
-  
+
-  
+
   background-color: white;
 }
 
@@ -628,7 +629,7 @@ body:not(.present-mode) {
   img {
     width: 200px;
   }
-  
+
   .artifact {
     display: block;
     width: 200px;

views/partials/space.html

@@ -12,7 +12,7 @@
 
     </button>
   </span>
-  
+
   <div class="btn-group light round" v-if="zones.length">
     <button class="btn btn-md btn-transparent btn-icon" v-on:click="go_to_previous_zone()" title="<%=__("Previous Zone")%>">
       <span class="icon icon-triangle-4-left"></span>
@@ -141,8 +141,8 @@
           <div class="progress" v-bind:style="{width: a.view.progress+'%'}"></div>
           <div class="progress-text">{{a.description}}</div>
 
-          <video v-if="a.mime == 'image/gif' && a.payload_alternatives && a.payload_alternatives.length > 0" preload autoplay loop>
+          <video v-if="a.mime == 'image/gif' && a.view.payload_alternatives && a.view.payload_alternatives.length > 0" preload autoplay loop>
-            <source v-for="rep in a.payload_alternatives" v-bind:src="rep.payload_uri" v-bind:type="rep.mime" />
+            <source v-for="rep in a.view.payload_alternatives" v-bind:src="rep.payload_uri" v-bind:type="rep.mime" />
           </video>
 
           <span v-if="a.view.link.length>0" class="link-wrapper">
@@ -151,10 +151,10 @@
         </div>
 
         <!-- video -->
-        <div v-if="a.view.major_type == 'video'" v-videoplayer="a" class="video" v-bind:style="a.view.inner_style + ';background-image: url('+a.view.thumbnail_uri+');'">
+        <div v-if="a.view.major_type == 'video'" v-videoplayer="a" class="video" v-bind:style="a.view.inner_style">
           <video preload="metadata" v-bind:poster="a.view.thumbnail_uri">
-            <source v-for="rep in a.payload_alternatives" v-bind:src="rep.payload_uri" v-bind:type="rep.mime" />
+            <source v-for="rep in a.view.payload_alternatives" v-bind:src="rep.payload_uri" v-bind:type="rep.mime" />
-            <source v-if="a.payload_uri && a.mime" v-bind:src="a.payload_uri" v-bind:type="a.mime" />
+            <source v-if="a.view.payload_uri && a.view.mime" v-bind:src="a.view.payload_uri" v-bind:type="a.view.mime" />
           </video>
 
           <div class="tl-controls">
@@ -181,11 +181,11 @@
         <div v-if="a.view.major_type == 'audio'" v-audioplayer="a" class="audio" v-bind:style="a.view.inner_style">
 
           <audio>
-            <source v-for="alt in a.payload_alternatives | orderBy 'mime' -1" v-bind:src="alt.payload_uri" v-bind:type="alt.mime"/>
+            <source v-for="alt in a.view.payload_alternatives | orderBy 'mime' -1" v-bind:src="alt.payload_uri" v-bind:type="alt.mime"/>
-            <source v-bind:src="a.payload_uri" v-bind:type="a.mime" v-if="a.payload_uri"/>
+            <source v-bind:src="a.view.payload_uri" v-bind:type="a.view.mime" v-if="a.view.payload_uri"/>
           </audio>
 
-          <div class="timeline" v-show="a.h>=64 && a.w>=170" v-bind:style="{'background-image': 'url(' + a.payload_thumbnail_web_uri +')'}">
+          <div class="timeline" v-show="a.h>=64 && a.w>=170" v-bind:style="{'background-image': 'url(' + a.view.thumbnail_uri +')'}">
             <div class="tl-current-time" v-bind:style="{width: a.player_view.current_time_float*100 + '%'}"></div>
             <div class="tl-inpoint" v-bind:style="{left: a.player_view.inpoint_float*100 + '%'}" v-if="a.player_view.inpoint_float>0.0"></div>
             <div class="tl-outpoint" v-bind:style="{left: a.player_view.outpoint_float*100 + '%'}"></div>
@@ -238,8 +238,8 @@
         </div>
 
         <!-- embed -->
-        <div v-if="a.view.major_type == 'oembed'" class="oembed" 
+        <div v-if="a.view.major_type == 'oembed'" class="oembed"
-          v-bind:style="{'background-image': 'url('+a.view.thumbnail_uri+')'}" 
+          v-bind:style="{'background-image': 'url('+a.view.thumbnail_uri+')'}"
           v-bind:class="{interactive:(a.view.interactive || present_mode)}">
           {{{a.view.oembed_html}}}
           <button class="btn btn-icon btn-primary btn-round interact" v-on:click="a.view.interactive=1" v-if="!a.view.interactive && !present_mode"><span class="icon icon-tool-pointer"></span></button>