Skip to content

Commit a041dda

Browse files
author
Bob Simons
committed
v2.21
Former-commit-id: b064a1821c6607906cf8e1ec5d765fb0f0ef09ca [formerly 1f734b10db1adf06fbe54c9bde29aab515fd96e0] [formerly 8f41754f7ba927b97eefb32a281c956f4a1f5325 [formerly cd04e31b5b21e93963833ee56fda8ce2084865a4]] Former-commit-id: 5eb93b0ffcde6ce4082a308b6d4c40c94b42e7e0 [formerly de4dc96c1a4fce3b414cabc84d154c7757811dbb] Former-commit-id: 2e328bffa1ddcef6948e261189924642a02b45fe Former-commit-id: 722977b77578c5dc4a1863c36dd37263063d0c79
1 parent 54a0239 commit a041dda

11 files changed

+142
-75
lines changed

WEB-INF/classes/gov/noaa/pfel/coastwatch/TestAll.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ public static void main(String args[]) throws Throwable {
125125
// String tFileName = "/u00/satellite/MH1/chla/1day/AQUA_MODIS.20220301.L3m.DAY.CHL.chlor_a.4km.NRT.nc";
126126
// String2.log(NcHelper.ncdump(tFileName, "-h"));
127127

128-
// DasDds.main(new String[]{"nesdisVHNnoaaSNPPnoaa20chlaDaily", "-verbose"});
128+
// DasDds.main(new String[]{"esrlNcepRe", "-verbose"});
129129

130130
// String2.log(EDDTableFromAsciiFiles.generateDatasetsXml("S://obisSubset/", ".*\\.csv",
131131
// "", "", 1, 2, ",", 1000000000, "", "", "", "", "", "", "myInfo", "myInstitution", "mySummary", "myTitle",
@@ -529,15 +529,15 @@ public static void main(String args[]) throws Throwable {
529529
// String2.log(NcHelper.ncdump("/u00/data/points/caricoos/181p1_historic.nc", "-v metaStationLatitude;metaStationLongitude"));
530530
/*
531531
s = EDDTableFromMultidimNcFiles.generateDatasetsXml(
532-
"/data/sarah/", ".*\\.nc", "",
533-
"TIME_PRE", //dims
532+
"/data/bodc/", "Cabot.*\\.nc", "",
533+
"", //dims
534534
1440,
535535
"", "", "", "", //pre, post, extract, varname
536536
true, //removeMVRows //often true
537537
"", //sort files by profile_time
538538
"", "", "", "",
539539
0, //standardizeWhat 1+2(numericTime)+256(catch numeric mv)+4096(units)
540-
"DEPTH,TIME_PRE", //treatDimensionsAs
540+
"", //treatDimensionsAs
541541
"", //cacheFromUrl /catalog.html
542542
null) + "\n";
543543
String2.setClipboardString(s); String2.log(s);

WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ public class EDStatic {
187187
* <br>2.18 released on 2022-02-23
188188
* <br>2.19 released on 2022-09-10
189189
* <br>2.20 released on 2022-09-30
190+
* <br>2.21 released on 2022-10-09
190191
*
191192
* For master branch releases, this will be a floating point
192193
* number with 2 decimal digits, with no additional text.
@@ -200,7 +201,7 @@ public class EDStatic {
200201
* A request to http.../erddap/version will return just the number (as text).
201202
* A request to http.../erddap/version_string will return the full string.
202203
*/
203-
public static String erddapVersion = "2.20"; //see comment above
204+
public static String erddapVersion = "2.21"; //see comment above
204205

205206
/**
206207
* This is almost always false.
@@ -4598,6 +4599,7 @@ public static void addIntroStatistics(StringBuilder sb) {
45984599
}
45994600
} catch (Exception e) {
46004601
}
4602+
sb.append("Number of active requests=" + activeRequests.size() + "\n");
46014603
}
46024604

46034605
/**

WEB-INF/classes/gov/noaa/pfel/erddap/util/EmailThread.java

+11-2
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ public class EmailThread extends Thread {
3636
private long lastStartTime = -1; //-1 if session not active
3737
public long lastSessionMillis = -1; //duration
3838

39-
public static int sleepMillis = 5000;
39+
public static long defaultSleepMillis = 15000;
40+
public static long sleepMillis = defaultSleepMillis;
4041

4142

4243
/**
@@ -59,7 +60,7 @@ public long elapsedTime() {
5960

6061

6162
/**
62-
* This repeatedly: sleeps for 5 seconds, then sends all pending emails.
63+
* This repeatedly: sleeps for sleepMillis, then sends all pending emails.
6364
*/
6465
public void run() {
6566

@@ -154,6 +155,9 @@ public void run() {
154155
}
155156
}
156157

158+
//reset sleepMillis because openEmailSession succeeded
159+
sleepMillis = defaultSleepMillis;
160+
157161
String2.log("%%% EmailThread session finished after email #" + (EDStatic.nextEmail - 1) +
158162
" at " + Calendar2.getCurrentISODateTimeStringLocalTZ());
159163

@@ -170,6 +174,11 @@ public void run() {
170174
" at " + Calendar2.getCurrentISODateTimeStringLocalTZ() + "\n" +
171175
MustBe.throwableToString(e));
172176

177+
//openEmailSession and other failures: wait longer before try again to avoid e.g.,
178+
// "jakarta.mail.AuthenticationFailedException: 454 4.7.0 Too many login attempts, please try again later."
179+
if (sleepMillis * 2 < 5 * Calendar2.MILLIS_PER_MINUTE)
180+
sleepMillis *= 2;
181+
173182
} finally {
174183
lastStartTime = -1;
175184
try {

download/AccessToPrivateDatasets.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ <h2><a class="selfLink" id="contact" href="#contact" rel="bookmark">Contact</a><
194194

195195
<br>&nbsp;
196196
<hr>
197-
<p>ERDDAP, Version 2.20
197+
<p>ERDDAP, Version 2.21
198198
<br><a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html">Disclaimers</a> |
199199
<a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html#privacyPolicy">Privacy Policy</a>
200200
<p>&nbsp;

download/EDDTableFromEML.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -606,7 +606,7 @@ <h2><a class="selfLink" id="contact" href="#contact" rel="bookmark">Contact</a><
606606

607607
<br>&nbsp;
608608
<hr>
609-
<p>ERDDAP, Version 2.20
609+
<p>ERDDAP, Version 2.21
610610
<br><a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html">Disclaimers</a> |
611611
<a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html#privacyPolicy">Privacy Policy</a>
612612

download/NCCSV.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -930,7 +930,7 @@ <h2><a class="selfLink" id="contact" href="#contact" rel="bookmark">Contact</a><
930930

931931
<br>&nbsp;
932932
<hr>
933-
<p>ERDDAP, Version 2.20
933+
<p>ERDDAP, Version 2.21
934934
<br><a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html">Disclaimers</a> |
935935
<a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html#privacyPolicy">Privacy Policy</a>
936936
</div>

download/NCCSV_1.00.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -859,7 +859,7 @@ <h2><a class="selfLink" id="contact" href="#contact" rel="bookmark">Contact</a><
859859

860860
<br>&nbsp;
861861
<hr>
862-
<p>ERDDAP, Version 2.20
862+
<p>ERDDAP, Version 2.21
863863
<br><a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html">Disclaimers</a> |
864864
<a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html#privacyPolicy">Privacy Policy</a>
865865
</div>

download/changes.html

+34-35
Original file line numberDiff line numberDiff line change
@@ -72,49 +72,41 @@ <h1 style="text-align:center;">ERDDAP Changes</h1>
7272

7373

7474
<h2><a class="selfLink" id="changes2.20" href="#changes2.20" rel="bookmark">Changes</a>
75-
in ERDDAP version 2.20 (released 2022-09-30)</h2>
75+
in ERDDAP version 2.21 (released 2022-10-09)</h2>
7676
<ul>
7777
<li><strong>New Features and Changes (for users):</strong>
7878
<ul>
7979
<li>(None)
8080
<br>&nbsp;
8181
</ul>
8282
<li><strong>Things ERDDAP Administrators Need to Know and Do:</strong>
83-
<ul>
84-
<li>TO DO: In [tomcat]/conf/context.xml, right before <kbd>&lt;/Context&gt;</kbd>,
85-
change the <kbd>Resources</kbd> tag (or add it if it isn't already there) to
86-
increase the <kbd>cacheMaxSize</kbd> setting from 80000 (or whatever it was) to 200000:
87-
<br><kbd>&lt;Resources cachingAllowed="true" cacheMaxSize="200000" /&gt;</kbd>
88-
<br>This avoids numerous warnings in catalina.out that all start with
89-
<br><kbd>"WARNING [main] org.apache.catalina.webresources.Cache.getResource Unable to add the resource at [/WEB-INF/classes/..."</kbd>
83+
<ul>
84+
<li>BUG FIX: Sometimes, the new email system attempted to log in too often,
85+
which caused Google Email servers to reject all future log in attempts. Now,
86+
the email system avoids this and related problems.
9087
<br>&nbsp;
91-
88+
</ul>
89+
</ul>
90+
91+
<!-- ****************************************************************************** -->
92+
93+
94+
<h2><a class="selfLink" id="changes2.20" href="#changes2.20" rel="bookmark">Changes</a>
95+
in ERDDAP version 2.20 (released 2022-09-30)</h2>
96+
<ul>
97+
<li><strong>New Features and Changes (for users):</strong>
98+
<ul>
99+
<li>(None)
100+
<br>&nbsp;
101+
</ul>
102+
<li><strong>Things ERDDAP Administrators Need to Know and Do:</strong>
103+
<ul>
92104
<li>IMPROVED: We re-enabled the old memory management system (Math2.ensureMemoryAvailable) and modified
93105
the new memory management system (EDStatic.shedThisRequest) to work better with it.
94-
<p>ERDDAP shouldn't ever crash or freeze up. If it does, one of the
95-
most likely causes is insufficient memory. You can monitor memory usage
96-
by looking at the status.html web page, which now includes a line like
97-
<br><kbd>0 gc calls, 0 requests shed, and 0 dangerousMemoryEmails since last major LoadDatasets</kbd>
98-
<br>(those are progressively more serious events)
99-
<br>and a <kbd>gc Calls</kbd> column in the table of statistics.
100-
You can tell how memory-stressed your ERDDAP is by watching these numbers
101-
in addition to the <kbd>MB inUse</kbd> columns.
102-
Higher numbers indicate more stress.
103-
<ul>
104-
<li><kbd>MB inUse</kbd> should always be less than half of your Xmx memory setting.
105-
Larger numbers are serious trouble.
106-
<li><kbd>gc calls</kbd> indicates the number of times ERDDAP called the
107-
garbage collector to try to alleviate high memory usage.
108-
<li><kbd>shed</kbd> indicates the number of incoming requests that
109-
were shed (with HTTP error number 503, Service Unavailable) because
110-
memory use was already too high. Ideally, no requests should be shed.
111-
<li><kbd>dangerousMemoryEmails</kbd> -
112-
If memory use becomes dangerously high, ERDDAP sends an email to the email addresses listed in
113-
emailEverythingToCSV (in setup.xml) with a list of the active user requests. As the email says,
114-
please forward these emails to erd.data at noaa.gov so we can use the information
115-
to improve future versions of ERDDAP.
106+
See <a rel="help"
107+
href="https://coastwatch.pfeg.noaa.gov/erddap/download/setup.html#MemoryStatus"
108+
>Memory Status</a> for details.
116109
<br>&nbsp;
117-
</ul>
118110

119111
<li>CHANGED: The default for &lt;ipAddressMaxRequests&gt; in datasets.xml
120112
was increased from 7 to 15. It's clear that some legitimate WMS clients
@@ -159,6 +151,12 @@ <h2><a class="selfLink" id="changes2.19" href="#changes2.19" rel="bookmark">Chan
159151
There are no other changes that you need to make to your ERDDAP
160152
installation related to this change. In other words, ERDDAP works
161153
as it did before.
154+
155+
<p>Don't forget to make the ERDDAP-related changes to Tomcat's server.xml and context.xml
156+
when you upgrade Tomcat. See ERDDAP's
157+
<a rel="help" href="https://coastwatch.pfeg.noaa.gov/erddap/download/setup.html#tomcat"
158+
>Tomcat installation instructions</a>.
159+
162160
<p>My impression of Java 17 is that it prefers more processing power and memory
163161
for long-running, larger applications like ERDDAP, so it works slightly slower
164162
than Java 8 with low power computers (e.g., 2 cores and minimal RAM)
@@ -168,10 +166,10 @@ <h2><a class="selfLink" id="changes2.19" href="#changes2.19" rel="bookmark">Chan
168166
<a rel="bookmark" href="https://www.howtogeek.com/668986/how-to-use-the-linux-top-command-and-understand-its-output/">top<img
169167
src="../images/external.png" alt=" (external link)"
170168
title="This link to an external website does not constitute an endorsement."></a>
171-
to check resource usage and consider giving ERDDAP more resources.
169+
to check resource usage and consider giving ERDDAP more resources, notably more memory.
172170
Memory is cheap! Most phones have more processors and memory than
173171
the servers that some of you are using to run ERDDAP!
174-
Thanks to Erin Turnbull.
172+
<br>Thanks to Erin Turnbull.
175173
<br>&nbsp;
176174
<li>TO DO: If you use ERDDAP to access Cassandra, for Cassandra, you need to keep using the version
177175
of Java that you were using for running the Cassandra.
@@ -181,6 +179,7 @@ <h2><a class="selfLink" id="changes2.19" href="#changes2.19" rel="bookmark">Chan
181179
consider changing to these settings in your datasets.xml file:<kbd>
182180
<br>&nbsp;&nbsp;&lt;nGridThreads&gt;3&lt;/nGridThreads&gt;
183181
<br>&nbsp;&nbsp;&lt;nTableThreads&gt;3&lt;/nTableThreads&gt; </kbd>
182+
<br>If your server has fewer resources, stick to "1" for both of those settings.
184183
<br>The nThreads systems for EDDGridFromFiles and EDDTableFromFiles were significantly improved.
185184
These changes led to a huge speed improvement (e.g., 2X speedup when nThreads is set to 2 or more)
186185
for the most challenging requests (when a large number of files must be processed to gather the results).
@@ -6953,7 +6952,7 @@ <h2><a class="selfLink" id="contact" href="#contact" rel="bookmark">Contact</a><
69536952

69546953
<br>&nbsp;
69556954
<hr>
6956-
<p>ERDDAP, Version 2.20
6955+
<p>ERDDAP, Version 2.21
69576956
<br><a href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html">Disclaimers</a> |
69586957
<a href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html#privacyPolicy">Privacy Policy</a>
69596958

download/grids.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -1248,7 +1248,7 @@ <h3><a class="selfLink" id="RemoteReplicationOfDatasets_Solutions" href="#Remote
12481248

12491249
<br>&nbsp;
12501250
<hr>
1251-
<p>ERDDAP, Version 2.20
1251+
<p>ERDDAP, Version 2.21
12521252
<br><a rel="help" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html">Disclaimers</a> |
12531253
<a rel="help" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html#privacyPolicy">Privacy Policy</a>
12541254

0 commit comments

Comments
 (0)