SAK-50846 Sitestats successive sitestats calculated incorrectly

sitestats/schema/db-tables/hsqldb/sitestats.sql

@@ -1,7 +1,7 @@
 create table SST_EVENTS (ID bigint generated by default as identity (start with 1), USER_ID varchar(99) not null, SITE_ID varchar(99) not null, EVENT_ID varchar(32) not null, EVENT_DATE date not null, EVENT_COUNT bigint not null, primary key (ID));
 create table SST_JOB_RUN (ID bigint generated by default as identity (start with 1), JOB_START_DATE timestamp, JOB_END_DATE timestamp, START_EVENT_ID bigint, END_EVENT_ID bigint, LAST_EVENT_DATE timestamp, primary key (ID));
 create table SST_PREFERENCES (ID bigint generated by default as identity (start with 1), SITE_ID varchar(99) not null, PREFS longvarchar not null, primary key (ID));
-create table SST_PRESENCES (ID bigint generated by default as identity (start with 1), SITE_ID varchar(99) not null, USER_ID varchar(99) not null, P_DATE date not null, DURATION bigint default 0 not null, LAST_VISIT_START_TIME timestamp default null, primary key (ID));
+create table SST_PRESENCES (ID bigint generated by default as identity (start with 1), SITE_ID varchar(99) not null, USER_ID varchar(99) not null, P_DATE date not null, DURATION bigint default 0 not null, LAST_VISIT_START_TIME timestamp default null, CURRENT_OPEN_SESSIONS int default 0 not null, primary key (ID));
 create table SST_REPORTS (ID bigint generated by default as identity (start with 1), SITE_ID varchar(99), TITLE varchar(255) not null, DESCRIPTION longvarchar, HIDDEN bit, REPORT_DEF longvarchar not null, CREATED_BY varchar(99) not null, CREATED_ON timestamp not null, MODIFIED_BY varchar(99), MODIFIED_ON timestamp, primary key (ID));
 create table SST_RESOURCES (ID bigint generated by default as identity (start with 1), USER_ID varchar(99) not null, SITE_ID varchar(99) not null, RESOURCE_REF varchar(255) not null, RESOURCE_ACTION varchar(12) not null, RESOURCE_DATE date not null, RESOURCE_COUNT bigint not null, primary key (ID));
 create table SST_SITEACTIVITY (ID bigint generated by default as identity (start with 1), SITE_ID varchar(99) not null, ACTIVITY_DATE date not null, EVENT_ID varchar(32) not null, ACTIVITY_COUNT bigint not null, primary key (ID));

sitestats/schema/db-tables/mysql/sitestats.sql

@@ -1,7 +1,7 @@
 create table SST_EVENTS (ID bigint not null auto_increment, USER_ID varchar(99) not null, SITE_ID varchar(99) not null, EVENT_ID varchar(32) not null, EVENT_DATE date not null, EVENT_COUNT bigint not null, primary key (ID));
 create table SST_JOB_RUN (ID bigint not null auto_increment, JOB_START_DATE datetime, JOB_END_DATE datetime, START_EVENT_ID bigint, END_EVENT_ID bigint, LAST_EVENT_DATE datetime, primary key (ID));
 create table SST_PREFERENCES (ID bigint not null auto_increment, SITE_ID varchar(99) not null, PREFS text not null, primary key (ID));
-create table SST_PRESENCES (ID bigint not null auto_increment, SITE_ID varchar(99) not null, USER_ID varchar(99) not null, P_DATE date not null, DURATION bigint default 0 not null, LAST_VISIT_START_TIME datetime default null, primary key (ID));
+create table SST_PRESENCES (ID bigint not null auto_increment, SITE_ID varchar(99) not null, USER_ID varchar(99) not null, P_DATE date not null, DURATION bigint default 0 not null, LAST_VISIT_START_TIME datetime default null, CURRENT_OPEN_SESSIONS int default 0 not null, primary key (ID));
 create table SST_REPORTS (ID bigint not null auto_increment, SITE_ID varchar(99), TITLE varchar(255) not null, DESCRIPTION longtext, HIDDEN bit, REPORT_DEF text not null, CREATED_BY varchar(99) not null, CREATED_ON datetime not null, MODIFIED_BY varchar(99), MODIFIED_ON datetime, primary key (ID));
 create table SST_RESOURCES (ID bigint not null auto_increment, USER_ID varchar(99) not null, SITE_ID varchar(99) not null, RESOURCE_REF varchar(255) not null, RESOURCE_ACTION varchar(12) not null, RESOURCE_DATE date not null, RESOURCE_COUNT bigint not null, primary key (ID));
 create table SST_SITEACTIVITY (ID bigint not null auto_increment, SITE_ID varchar(99) not null, ACTIVITY_DATE date not null, EVENT_ID varchar(32) not null, ACTIVITY_COUNT bigint not null, primary key (ID));

sitestats/schema/db-tables/oracle/sitestats.sql

@@ -1,7 +1,7 @@
 create table SST_EVENTS (ID number(19,0) not null, USER_ID varchar2(99 char) not null, SITE_ID varchar2(99 char) not null, EVENT_ID varchar2(32 char) not null, EVENT_DATE date not null, EVENT_COUNT number(19,0) not null, primary key (ID));
 create table SST_JOB_RUN (ID number(19,0) not null, JOB_START_DATE timestamp, JOB_END_DATE timestamp, START_EVENT_ID number(19,0), END_EVENT_ID number(19,0), LAST_EVENT_DATE timestamp, primary key (ID));
 create table SST_PREFERENCES (ID number(19,0) not null, SITE_ID varchar2(99 char) not null, PREFS clob not null, primary key (ID));
-create table SST_PRESENCES (ID number(19,0) not null, SITE_ID varchar2(99 char) not null, USER_ID varchar2(99 char) not null, P_DATE date not null, DURATION number(19,0) default 0 not null, LAST_VISIT_START_TIME timestamp default null, primary key (ID));
+create table SST_PRESENCES (ID number(19,0) not null, SITE_ID varchar2(99 char) not null, USER_ID varchar2(99 char) not null, P_DATE date not null, DURATION number(19,0) default 0 not null, LAST_VISIT_START_TIME timestamp default null, CURRENT_OPEN_SESSIONS number(10,0) default 0 not null, primary key (ID));
 create table SST_REPORTS (ID number(19,0) not null, SITE_ID varchar2(99 char), TITLE varchar2(255 char) not null, DESCRIPTION clob, HIDDEN number(1,0), REPORT_DEF clob not null, CREATED_BY varchar2(99 char) not null, CREATED_ON timestamp not null, MODIFIED_BY varchar2(99 char), MODIFIED_ON timestamp, primary key (ID));
 create table SST_RESOURCES (ID number(19,0) not null, USER_ID varchar2(99 char) not null, SITE_ID varchar2(99 char) not null, RESOURCE_REF varchar2(255 char) not null, RESOURCE_ACTION varchar2(12 char) not null, RESOURCE_DATE date not null, RESOURCE_COUNT number(19,0) not null, primary key (ID));
 create table SST_SITEACTIVITY (ID number(19,0) not null, SITE_ID varchar2(99 char) not null, ACTIVITY_DATE date not null, EVENT_ID varchar2(32 char) not null, ACTIVITY_COUNT number(19,0) not null, primary key (ID));

sitestats/sitestats-api/src/java/org/sakaiproject/sitestats/api/SitePresence.java

@@ -41,4 +41,10 @@ public interface SitePresence extends Stat, Comparable<SitePresence> {
 
 	/** Set (temporary) last visit start time */
 	public void setLastVisitStartTime(Date lastVisitStartTime);
+
+	/** Get previus open sessions */
+	public int getCurrentOpenSessions();
+
+	/** Set current open sessions */
+	public void setCurrentOpenSessions(int currentOpenSessions);
 }

sitestats/sitestats-impl-hib/src/java/org/sakaiproject/sitestats/impl/SitePresenceImpl.java

@@ -39,6 +39,7 @@ public class SitePresenceImpl implements SitePresence, Serializable {
 	private Date date;
 	private long duration;
 	private Date lastVisitStartTime;
+	private int currentOpenSessions;
 
 	@Override
 	public int compareTo(SitePresence other) {
@@ -67,7 +68,8 @@ public boolean equals(Object o) {
 				&& date.equals(other.getDate())
 				&& duration == other.getDuration()
 				&& getCount() == other.getCount()
-				&& lastVisitStartTime == other.getLastVisitStartTime();
+				&& lastVisitStartTime == other.getLastVisitStartTime()
+				&& currentOpenSessions == other.getCurrentOpenSessions();
 	}
 
 	@Override
@@ -79,12 +81,13 @@ public int hashCode() {
 				+ this.getUserId().hashCode()
 				+ this.getDate().hashCode()
 				+ duration
-				+ this.getLastVisitStartTime();
+				+ this.getLastVisitStartTime()
+				+ currentOpenSessions;
 		return hashStr.hashCode();
 	}
 
 	public String toString(){
-		return siteId + " : " + userId + " : " + date + " : " + duration + " (" + lastVisitStartTime + ")";
+		return siteId + " : " + userId + " : " + date + " : " + duration + " (" + lastVisitStartTime + ") : " + currentOpenSessions;
 	}
 
 	public long getDuration() {
@@ -143,4 +146,11 @@ public long getCount() {
 		return duration;
 	}
 
+	public int getCurrentOpenSessions() {
+		return currentOpenSessions;
+	}
+
+	public void setCurrentOpenSessions(int currentOpenSessions) {
+		this.currentOpenSessions = currentOpenSessions;
+	}
 }

sitestats/sitestats-impl-hib/src/java/org/sakaiproject/sitestats/impl/hbm/SitePresenceImpl.hbm.xml

@@ -25,5 +25,8 @@
 		<property name="lastVisitStartTime" type="timestamp" not-null="false">
 			<column name="LAST_VISIT_START_TIME" not-null="false" default="null" />
 		</property>
+		<property name="currentOpenSessions" type="int" not-null="true">
+			<column name="CURRENT_OPEN_SESSIONS" not-null="true" default="0" />
+		</property>
 	</class>
 </hibernate-mapping>

sitestats/sitestats-impl/src/java/org/sakaiproject/sitestats/impl/PresenceConsolidation.java

@@ -29,9 +29,11 @@
 import java.util.function.Function;
 
 import org.sakaiproject.sitestats.api.presence.Presence;
+import lombok.extern.slf4j.Slf4j;
 
 import lombok.NonNull;
 
+@Slf4j
 public class PresenceConsolidation {
 
 
@@ -54,6 +56,7 @@ public boolean add(@NonNull Presence presence) {
 
         if (records.contains(additialRecord)) {
             // Record is already present in list, leave unchanged
+            log.debug("Record already present: {}", additialRecord);
             return false;
         }
 
@@ -73,10 +76,14 @@ public boolean add(@NonNull Presence presence) {
             }
         }
 
+        log.debug("First overlapping record: {}", firstOverlappingRecord);
+        log.debug("Last overlapping record: {}", lastOverlappingRecord);
+
         // If we have no intersections, just add the new record
         if (firstOverlappingRecord == null && lastOverlappingRecord == null) {
             records.add(additialRecord);
             sort();
+            log.debug("Added new record: {}", additialRecord);
             return true;
         }
 
@@ -85,6 +92,7 @@ public boolean add(@NonNull Presence presence) {
             Presence onlyIntersectingRecord = firstOverlappingRecord;
 
             if (additialRecord.isWithin(onlyIntersectingRecord)) {
+                log.debug("New record is within the existing record: {}", additialRecord);
                 return false;
             }
 
@@ -99,6 +107,7 @@ public boolean add(@NonNull Presence presence) {
                 onlyIntersectingRecord.setEnd(additionalRecordEnd);
             }
 
+            log.debug("Extended existing record: {}", onlyIntersectingRecord);
             return true;
         }
 
@@ -113,6 +122,7 @@ public boolean add(@NonNull Presence presence) {
 
             PresenceRecord mergedRecord = merge(additialRecord, firstOverlappingRecord, lastOverlappingRecord);
             records.add(firstIndex, mergedRecord);
+            log.debug("Merged records into: {}", mergedRecord);
             return true;
         }
 
@@ -155,13 +165,18 @@ public Map<Instant, PresenceConsolidation> mapByDay() {
         // Split cross day records
         for (int i = 0; i < records.size(); i++) {
             PresenceRecord record = records.get(i);
+            log.debug("Processing record: {}", record);
 
             if (record.isCrossDay()) {
                 Instant begin = record.getBegin();
                 Instant end = record.getEnd();
                 Instant beginDay = toDay(begin);
                 Instant endDay = toDay(end);
 
+                log.debug("Record crosses day: {}", record);
+                log.debug("Begin: {}, End: {}", begin, end);
+                log.debug("Begin day: {}, End day: {}", beginDay, endDay);
+
                 for (Instant day = beginDay;
                         day.isBefore(end);
                         day = day.plus(1, ChronoUnit.DAYS)) {
@@ -170,16 +185,20 @@ public Map<Instant, PresenceConsolidation> mapByDay() {
                     Instant sliceBegin = day.equals(beginDay) ? begin : day;
                     Instant sliceEnd = day.equals(endDay) ? end : day.plus(1, ChronoUnit.DAYS);
 
+                    log.debug("Creating slice: Begin: {}, End: {}", sliceBegin, sliceEnd);
+
                     PresenceRecord presenceSlice = PresenceRecord.builder()
                             .begin(sliceBegin)
                             .end(sliceEnd)
                             .build();
 
                     // Get or create presence consolidation for day and add record
                     recordsByDay.computeIfAbsent(day, consolidationFacrory).add(presenceSlice);
+                    log.debug("Added slice to day: {}, Slice: {}", day, presenceSlice);
                 }
             } else {
                 recordsByDay.computeIfAbsent(record.getDay(), consolidationFacrory).add(record);
+                log.debug("Added record to day: {}, Record: {}", record.getDay(), record);
             }
         }
 

sitestats/sitestats-impl/src/java/org/sakaiproject/sitestats/impl/SitePresenceRecord.java

@@ -31,8 +31,13 @@ public class SitePresenceRecord extends PresenceRecord {
 
     private String siteId;
     private String userId;
+    private boolean originallyEnding;
 
 
+    public boolean isOriginallyEnding() {
+        return originallyEnding;
+    }
+
     public SitePresenceKey getKey() {
         return SitePresenceKey.from(this);
     }

sitestats/sitestats-impl/src/java/org/sakaiproject/sitestats/impl/StatsAggregateJobImpl.java

@@ -153,7 +153,7 @@ public void execute(JobExecutionContext context) throws JobExecutionException {
 		// check for SAKAI_EVENT.CONTEXT column
 		try{
 			checkForContextColumn();
-			log.debug("SAKAI_EVENT.CONTEXT exists? "+isEventContextSupported);
+			log.debug("SAKAI_EVENT.CONTEXT exists? {}", isEventContextSupported);
 		}catch(SQLException e1){
 			log.warn("Unable to check existence of SAKAI_EVENT.CONTEXT", e1);
 		}
@@ -310,16 +310,20 @@ private String startJob() throws SQLException {
 						if(firstEventIdProcessedInBlock == -1)
 							firstEventIdProcessedInBlock = lastProcessedEventId;
 						processedCounter++;
+						if (event == "pres.end" || event == "pres.begin") {
+							log.debug("Processed event: {}, date: {}, sessionUser: {}, sessionId: {}, eventId: {}", event, date, sessionUser, sessionId, lastProcessedEventId);
+						}
 					}catch(Exception e){
 						if(log.isDebugEnabled())
-							log.debug("Ignoring "+event+", "+ref+", "+date+", "+sessionUser+", "+sessionId+" due to: "+e.toString());
+							log.debug("Ignoring {}, {}, {}, {}, {} due to: {}", event, ref, date, sessionUser, sessionId, e.toString());
 					}
 					counter++;
 				}
 				rs.close();
 
 				// If we didn't see a single event, time to break out and wrap up this job
 				if (counter < 1) {
+					log.debug("No events found in this block, breaking out of the loop.");
 					break;
 				}
 
@@ -335,6 +339,7 @@ private String startJob() throws SQLException {
 						jobRun.setLastEventDate(lastEventDateWithSuccess);
 						jobRun.setJobEndDate(new Date(System.currentTimeMillis()));
 						saveJobRun(jobRun);
+						log.debug("Successfully processed events up to eventId: {}", lastProcessedEventIdWithSuccess);
 					}else{
 						returnMessage = "An error occurred while processing/persisting events to db. Please check your logs, fix possible problems and re-run this job (will start after last successful processed event).";
 						log.error(returnMessage);
@@ -456,7 +461,7 @@ public long collectPastSiteEvents(String siteId, Date initialDate, Date finalDat
 					count++;				
 				}catch(Exception e){
 					if(log.isDebugEnabled())
-						log.debug("Ignoring "+event+", "+ref+", "+date+", "+sessionUser+", "+sessionId+" due to: "+e.toString());
+						log.debug("Ignoring {}, {}, {}, {}, {} due to: {}", event, ref, date, sessionUser, sessionId, e.toString());
 				}
 			}