Merge branch 'main' of https://github.com/richardmleggett/MARTi

bin/marti_engine_options.txt

@@ -42,3 +42,9 @@ BlastProcess
     MaxE:0.001
     MaxTargetSeqs:100
     BlastThreads:2
+
+CentrifugeProcess
+    Name:ref_prok
+    Database:/Users/leggettr/Documents/Databases/centrifuge/p_2018_4_15/p_compressed
+    CentrifugeThreads:1
+    UseToClassify

engine/src/main/java/uk/ac/earlham/lcaparse/LCAFileParser.java

@@ -178,7 +178,7 @@ public int parseFile(String filename) {
         return hitsByQuery.size();
     }
 
-    public int removePoorAlignments() {
+    public ArrayList<String> removePoorAlignments() {
         Set<String> keys = hitsByQuery.keySet();
         ArrayList<String> idsToRemove = new ArrayList<String>();
         for (String queryName : keys) {
@@ -194,7 +194,7 @@ public int removePoorAlignments() {
             hitsByQuery.remove(idsToRemove.get(i));
         }
 
-        return idsToRemove.size();
+        return idsToRemove;
     }
 
     /**

engine/src/main/java/uk/ac/earlham/marti/amr/AMRResults.java

@@ -20,6 +20,7 @@
 import uk.ac.earlham.marti.core.MARTiEngineOptions;
 import javax.json.*;
 import javax.json.stream.JsonGenerator;
+import uk.ac.earlham.lcaparse.TaxonomyNode;
 import uk.ac.earlham.marti.core.MARTiEngine;
 
 /**
@@ -145,8 +146,16 @@ public void writeJSON(int cn) {
                 if(id == -2l){
                     speciesName = "Unassigned";
                 }
+
+                short rank = 0;
+                TaxonomyNode node = taxonomy.getNodeFromTaxonId(id);
+                if(node != null) {
+                    rank = node.getSimplifiedRank();
+                }
+
                 speciesBuilder.add("name", speciesName);
                 speciesBuilder.add("ncbiID", id);
+                speciesBuilder.add("rank", rank);
                 JsonObjectBuilder speciesCountBuilder = Json.createObjectBuilder();
                 JsonObjectBuilder plasmidCountBuilder = Json.createObjectBuilder();
                 cumulativeCount = 0;

engine/src/main/java/uk/ac/earlham/marti/classify/ReadClassifier.java

@@ -228,19 +228,25 @@ public synchronized void checkForFilesToClassify() {
                                 String perReadFilename = f.getClassifierPrefix() + "_perread.txt";
                                 options.getLog().println("Got LCAFileParse instance, now parsing");
                                 int readsWithHits = pfp.parseFile(f.getBlastFile());
-                                options.getLog().println("Parsed... now removing poor alignments");
-                                int readsRemoved = pfp.removePoorAlignments();
-
                                 long totalBpWithHits = 0l;
                                 Set<String> hits = pfp.getHitsByQuery().keySet();
                                 for(String query : hits) {
                                     totalBpWithHits += options.getReadStatistics().getReadLength(barcode, query, true);
                                 }
+
+                                options.getLog().println("Parsed... now removing poor alignments");
+                                ArrayList<String> queriesToRemove = pfp.removePoorAlignments();
+                                int readsRemoved = queriesToRemove.size();
+
+                                long bpRemoved = 0l;
+                                for(String query : queriesToRemove) {
+                                    bpRemoved += options.getReadStatistics().getReadLength(barcode, query, true);
+                                }
 
                                 options.getLog().println("Adding to reads classified");
                                 md.addToReadsClassified(readsWithHits, totalBpWithHits);
                                 options.getLog().println("Marking poor alignments");
-                                md.markPoorAlignments(readsRemoved);
+                                md.markPoorAlignments(readsRemoved, bpRemoved);
                                 options.getLog().println("Registering chunks");
                                 md.registerChunkAnalysed(f.getQueryFile());
 

engine/src/main/java/uk/ac/earlham/marti/core/MARTiResults.java

@@ -17,8 +17,6 @@
 import java.io.Reader;
 import java.io.StringWriter;
 import java.io.Writer;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.time.temporal.ChronoUnit;
@@ -34,7 +32,6 @@
 import uk.ac.earlham.lcaparse.TaxonomyNode;
 import javax.json.*;
 import javax.json.stream.JsonGenerator;
-import uk.ac.earlham.lcaparse.LCAHit;
 import uk.ac.earlham.marti.centrifuge.CentrifugeClassifierItem;
 
 /**
@@ -144,6 +141,7 @@ private void outputNode(int bc, TaxonomyNode n, JsonObjectBuilder treeBuilder, P
             // If root node, need to add unclassified count
             if (n.getId() == 1) {
                 summedCount += options.getSampleMetaData(bc).getReadsUnclassified();
+                summedYield  += options.getSampleMetaData(bc).getYieldUnclassified();
             }            
 
             rank = mr.getRankFromString(ncbiRankString);

engine/src/main/java/uk/ac/earlham/marti/core/SampleMetaData.java

@@ -53,11 +53,13 @@ public class SampleMetaData {
     private int readsClassified = 0;
     private int readsWithPoorAlignments = 0;
     private int readsAnalysed = 0;
+    private long bpAnalysed = 0;
     private long totalInputBp = 0;
     private long totalClassifiedBp = 0;
     private int countByQuality[] = new int[51];
     private double totalQuality = 0;
     private Hashtable<String,Integer> chunkCounts = new Hashtable<String,Integer>();
+    private Hashtable<String,Long> chunkYields = new Hashtable<String,Long>();
     private long startTime = System.nanoTime();
     private int lastChunkAnalysedTime = 0;
     private ArrayList<Integer> chunkAnalysedTimings = new ArrayList<Integer>();
@@ -105,21 +107,29 @@ public synchronized void registerNewInputRead(int bp, double meanQ, boolean pass
         }
     }  
 
-    public synchronized void registerFilteredFastaChunk(String fastaFilename, int count) {
+    public synchronized void registerFilteredFastaChunk(String fastaFilename, int count, long yield) {
         options.getLog().println("Registering filtered chunk "+fastaFilename + " with "+count+" reads");
         readsPassedFilterByChunk += count;
         if (chunkCounts.containsKey(fastaFilename)) {
             options.getLog().printlnLogAndScreen("Error: filename "+fastaFilename+" already seen.");
         } else {
             chunkCounts.put(fastaFilename, count);
         }
+        if(chunkYields.contains(fastaFilename)) {
+            options.getLog().printlnLogAndScreen("Error: filename "+fastaFilename+" already seen.");
+        } else {
+            chunkYields.put(fastaFilename, yield);
+        }
+
     }
 
     public synchronized void registerChunkAnalysed(String fastaFilename) {
-        if (chunkCounts.containsKey(fastaFilename)) {
+        if (chunkCounts.containsKey(fastaFilename) && chunkYields.containsKey(fastaFilename)) {
             int count = chunkCounts.get(fastaFilename);
+            long yield = chunkYields.get(fastaFilename);
             options.getLog().println("Chunk analysed "+fastaFilename+" with "+count+" reads");
             readsAnalysed += count;
+            bpAnalysed += yield;
             lastChunkAnalysedTime = this.getMinutesSinceStart();
             chunkAnalysedTimings.add(lastChunkAnalysedTime);
         } else {
@@ -132,8 +142,9 @@ public synchronized void addToReadsClassified(int n, long bp) {
         totalClassifiedBp += bp;
     }
 
-    public synchronized void markPoorAlignments(int n) {
+    public synchronized void markPoorAlignments(int n, long bp) {
         readsClassified-=n;
+        totalClassifiedBp -= bp;
         readsWithPoorAlignments+=n;
     }
 
@@ -148,7 +159,7 @@ public int getReadsUnclassified() {
     }
 
     public long getYieldUnclassified() {
-        return totalInputBp - totalClassifiedBp;
+        return bpAnalysed - totalClassifiedBp;
     }
 
     public synchronized void writeSampleJSON(boolean martiComplete) {
@@ -227,8 +238,10 @@ public synchronized void writeSampleJSON(boolean martiComplete) {
         sampleObjectBuilder.add("readsPassedFilter", readsPassedFilter);
         sampleObjectBuilder.add("readsWithClassification", readsClassified);
         sampleObjectBuilder.add("readsUnclassified", getReadsUnclassified());
+        sampleObjectBuilder.add("classifiedYield", totalClassifiedBp);
+        sampleObjectBuilder.add("unclassifiedYield", getYieldUnclassified());
         sampleObjectBuilder.add("readsWithPoorAlignments", readsWithPoorAlignments);
-        sampleObjectBuilder.add("readsAnalysed", readsAnalysed);       
+        sampleObjectBuilder.add("readsAnalysed", readsAnalysed);    
         sampleObjectBuilder.add("sequencingStatus", "Complete");
         sampleObjectBuilder.add("martiStatus", martiComplete ? "Complete":"Processing");
         sampleObjectBuilder.add("analysis", analysisObjectBuilder);

engine/src/main/java/uk/ac/earlham/marti/filter/ReadFilterSample.java

@@ -30,6 +30,7 @@ public class ReadFilterSample {
     PrintWriter pwFasta = null;
     private int chunkNumber = -1;
     private int readCountInChunk = 0;
+    private long bpInChunk = 0;
     private String currentFastqChunkFilename = null;
     private String currentFastaChunkFilename = null;
     private ArrayList<Integer> allReadLengths = new ArrayList<Integer>();
@@ -200,7 +201,7 @@ private synchronized void endChunks() {
 
             options.getLog().println("Moves complete.");
 
-            metaData.registerFilteredFastaChunk(currentFastaChunkFilename, readCountInChunk);
+            metaData.registerFilteredFastaChunk(currentFastaChunkFilename, readCountInChunk, bpInChunk);
             metaData.writeSampleJSON(false);
 
             options.getLog().println("Now to add to pending pair list");
@@ -211,6 +212,7 @@ private synchronized void endChunks() {
             options.getLog().println("Reads filtered from chunk = "+readsFilteredFromChunk);
 
             readCountInChunk = 0;
+            bpInChunk = 0l;
 
             if (options.getStopProcessingAfter() > 0) {
                 if (writtenReadLengths.size() >= options.getStopProcessingAfter()) {
@@ -351,6 +353,7 @@ public void processFile(String fastqPathname) {
                                 }
 
                                 readCountInChunk++;
+                                bpInChunk += seq.length();
                                 writtenReadLengths.add(seq.length());
                                 chunkReadLengths.add(seq.length());
                                 readStatistics.addReadLength(barcode, readID,seq.length(), true);

gui/UI/index.js

@@ -155,7 +155,7 @@ if (serverOptions["https"].toLowerCase() === 'true') {
 
 const restrictedMode = argv.r || false;
 
-const martiGuiVersion = "0.19.6";
+const martiGuiVersion = "0.19.7";
 
 if (argv.v || argv.version) {
   console.log(martiGuiVersion);

gui/UI/indexNode.html

@@ -406,6 +406,25 @@ <h5 class="modal-title">You need to select a sample!</h5>
   </div>
 </div>
 
+<!-- New Analysis Modal -->
+<div class="modal" id="newAnalysisModal" tabindex="-1" role="dialog">
+<div class="modal-dialog modal-dialog-centered" role="document">
+  <div class="modal-content">
+    <div class="modal-header">
+      <h5 class="modal-title">Page locked</h5>
+      <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+        <span aria-hidden="true">&times;</span>
+      </button>
+    </div>
+    <div class="modal-body">
+      <p>Download and install MARTi to access the new analysis page.</p>
+    </div>
+    <div class="modal-footer">
+      <button type="button" class="btn btn-secondary" data-dismiss="modal">Close</button>
+    </div>
+  </div>
+</div>
+</div>
 
 </body>
 
@@ -487,14 +506,22 @@ <h5 class="modal-title">You need to select a sample!</h5>
     });
 
     $("#new-item").click(function () {
-      activeSidebarIcon(this);
-      currentPage = "New";
-      $("h1#pageTitle").text("New analysis");
-      urlFormat();
-      $("#response").load("/new.html", function() {
-      $("html, body").animate({ scrollTop: "0px" });
-      initialiseNewPage();
-      });
+
+      if (restrictedMode){
+          $('#newAnalysisModal').modal('show')
+      } else {
+          activeSidebarIcon(this);
+          currentPage = "New";
+          $("h1#pageTitle").text("New analysis");
+          urlFormat();
+          $("#response").load("/new.html", function() {
+          $("html, body").animate({ scrollTop: "0px" });
+          initialiseNewPage();
+          });
+      }
+
+
+
     });
 
 dashboardPageUnlocked = false;

gui/UI/public/dashboard.html

@@ -410,6 +410,12 @@ <h6 class="m-0 font-weight-bold text-primary">Donut</h6>
               <input type="range" class="form-control-range" name="dashboardTaxaDonutTopN" min="1" max="20" value="10">
             </div>
 
+            <div>
+              <label> Unclassified node:</label>
+              <label><input type="radio" name="dashboardTaxaDonutUnclassified" value="show" checked>  Show</label>
+              <label><input type="radio" name="dashboardTaxaDonutUnclassified" value="hide" >  Hide</label>
+            </div>
+
             </form>
 
             <div class="dropdown-divider"></div>
@@ -645,8 +651,12 @@ <h6 class="m-0 font-weight-bold text-primary">Treemap</h6>
             <div class="form-group">
               <label for="dashboardTreeMapColourBy">Colour by:</label>
               <select class="form-control form-control-sm" id="dashboardTreeMapColourBy">
-                <option>Phylum</option>
                 <option>Domain</option>
+                <option selected>Phylum</option>
+                <option>Class</option>
+                <option>Order</option>
+                <option>Family</option>
+                <option>Genus</option>
               </select>
             </div>
 

gui/UI/public/js/dashboard.js

@@ -477,6 +477,13 @@ socket.on('dashboard-meta-response', function(data) {
 
   $("#dashboardInfoCardYieldBasecalled").text(totalYieldFormatter(dashboardSampleData.yieldBases));
 
+  let classifiedYield = "-";
+  if (dashboardSampleData.hasOwnProperty("classifiedYield")){
+    classifiedYield = totalYieldFormatter(parseInt(dashboardSampleData.classifiedYield));
+  }
+
+  $("#dashboardInfoCardYieldClassified").text(classifiedYield);
+
   let readsUnclassified = dashboardSampleData.readsAnalysed - dashboardSampleData.readsWithClassification;
 
   $("#dashboardInfoCardReadsUnclassified").text(thousandsSeparators(readsUnclassified));
@@ -639,10 +646,14 @@ var donutLeaves = [];
 var donutTaxaAtRank = [];
 
 function taxaAtRank(d) {
-
   if (d.rank < taxonomicRankSelected) {
     if(taxonomicRankSelected == 10){
-      donutTaxaAtRank.push(d);
+      if (d.name == "unclassified" && dashboardTaxaDonutUnclassified == "hide"){
+
+      } else {
+        donutTaxaAtRank.push(d);
+      }
+
     };
     if (d.children) {
       d.children.forEach(function(c){

gui/UI/public/js/dashboardAmrHitsDonut.js

@@ -211,13 +211,21 @@ function generateAmrHitsSpeciesArray(data){
           if (geneCountAtChunk > 0) {
             let speciesDropdownListIndex = findWithAttr(speciesDropdownList, "ncbiID", taxa.ncbiID);
             if (speciesDropdownListIndex == -1){
-              let speciesInformationIndex = findWithAttr(donutNodes, "ncbiID", taxa.ncbiID);
-              if (speciesInformationIndex != -1) {
-                let speciesInfo = donutNodes[speciesInformationIndex];
-                if (speciesInfo.rank == 8) {
-                  speciesDropdownList.push({ncbiID:taxa.ncbiID, name:taxa.name})
+              if (taxa.hasOwnProperty("rank")) {
+                if (taxa.rank >= 8) {
+                  speciesDropdownList.push({ncbiID:taxa.ncbiID, name:taxa.name});
+                }
+              } else {
+                let speciesInformationIndex = findWithAttr(donutNodes, "ncbiID", taxa.ncbiID);
+                if (speciesInformationIndex != -1) {
+                  let speciesInfo = donutNodes[speciesInformationIndex];
+                  if (speciesInfo.rank >= 8) {
+                    speciesDropdownList.push({ncbiID:taxa.ncbiID, name:taxa.name})
+                  }
                 }
               }
+
+
             }
           }
         }

gui/UI/public/js/dashboardDonut.js

@@ -1,5 +1,5 @@
 
-var radius, pie, arc, outerArc, key;
+var radius, pie, arc, outerArc, key, dashboardTaxaDonutUnclassified;
 
 function initialiseDashboardDonut() {
 
@@ -50,7 +50,12 @@ key = function(d) {
 
       d3.selectAll(".dashboard-taxa-donut-top-n").text(dashboardTaxaDonutTopN);
 
+      d3.selectAll("input[name='dashboardTaxaDonutUnclassified']").on("change", function() {
+        dashboardTaxaDonutUnclassified = this.value;
+        globUpdate(globDonutData);
+      });
 
+      dashboardTaxaDonutUnclassified = "show";
 
 };