Skip to content

Commit da16148

Browse files
meiMinglemeiMingle
committed
Site updated: 2024-03-13 22:50:43
1 parent 2d530f4 commit da16148

File tree

14 files changed

+265
-46
lines changed

14 files changed

+265
-46
lines changed

‎2022/09/13/maven项目构建加速（二）辅助工具篇/index.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ <h1>maven项目构建加速（二）辅助工具篇</h1>
183183

184184
<span class="tag">
185185

186-
<a href="/tags/maven/" style="color: #ffa2c4">maven</a>
186+
<a href="/tags/maven/" style="color: #03a9f4">maven</a>
187187
</span>
188188

189189
</span>

‎2022/11/11/IntelliJ-IDEA常用配置/index.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ <h1>IntelliJ IDEA常用配置</h1>
183183

184184
<span class="tag">
185185

186-
<a href="/tags/IDEA/" style="color: #ffa2c4">IDEA</a>
186+
<a href="/tags/IDEA/" style="color: #03a9f4">IDEA</a>
187187
</span>
188188

189189
</span>

‎2024/03/13/数据库批量INSERT优化/index.html

Lines changed: 223 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -183,20 +183,20 @@ <h1>数据库批量Insert优化</h1>
183183

184184
<span class="tag">
185185

186-
<a href="/tags/SQL%E4%BC%98%E5%8C%96/" style="color: #ffa2c4">SQL优化</a>
186+
<a href="/tags/SQL%E4%BC%98%E5%8C%96/" style="color: #00a596">SQL优化</a>
187187
</span>
188188

189189
<span class="tag">
190190

191-
<a href="/tags/SQL%E5%90%88%E5%B9%B6/" style="color: #00bcd4">SQL合并</a>
191+
<a href="/tags/SQL%E5%90%88%E5%B9%B6/" style="color: #03a9f4">SQL合并</a>
192192
</span>
193193

194194
</span>
195195

196196
</div>
197197

198198
<div class="content" v-pre>
199-
<p>最近在做数据备份恢复时，数据库有一个限制，就是每小时只能执行几万条SQL语句。为了提高数据恢复效率，最好的办法就是将多个INSERT语句合并。假设我有30000条数据，那么将n个INSERT语句合并成一条执行，n取值为多少时，插入效率最高呢？</p>
199+
<p>最近在做数据备份恢复时，数据库有一个限制，就是每小时只能执行几万条SQL语句。为了提高数据恢复效率，最好的办法就是将多个INSERT语句合并。假设我有30000条数据，那么将x个INSERT语句合并成一条执行，x取值为多少时，插入效率最高呢？</p>
200200
<h1 id="一、准备环境和数据"><a href="#一、准备环境和数据" class="headerlink" title="一、准备环境和数据"></a>一、准备环境和数据</h1><p>数据库：Mysql8.0.33</p>
201201
<p>建表语句：</p>
202202
<pre><code class="mysql">-- ----------------------------
@@ -214,8 +214,227 @@ <h1 id="一、准备环境和数据"><a href="#一、准备环境和数据" clas
214214
) ENGINE = InnoDB CHARACTER SET = utf8mb3 COLLATE = utf8mb3_general_ci ROW_FORMAT = Dynamic;
215215
</code></pre>
216216
<p>INSERT 语句：</p>
217-
<p><a href="./%E6%95%B0%E6%8D%AE%E5%BA%93%E6%89%B9%E9%87%8FINSERT%E4%BC%98%E5%8C%96/INSERT.sql">INSERT语句</a></p>
217+
<p><a href="./INSERT.sql">INSERT语句</a></p>
218+
<p>以下是用来合并SQL的代码，使x分别取值2、3、4、5、6、7、8、9、10、11、12、13、14、15、16、17、18、19、20</p>
219+
<p>得到20个SQL文件</p>
220+
<pre><code class="java">package org.example;
218221

222+
import cn.hutool.core.io.FileUtil;
223+
import cn.hutool.core.io.IORuntimeException;
224+
import cn.hutool.core.io.IoUtil;
225+
import cn.hutool.core.text.StrBuilder;
226+
227+
import java.io.*;
228+
import java.nio.charset.StandardCharsets;
229+
230+
231+
/**
232+
* 主要功能：将多个insert语句合并成一个
233+
* 参考了这个代码 cn.hutool.core.text.csv.CsvParser
234+
*/
235+
public class Main &#123;
236+
237+
public static void main(String[] args) &#123;
238+
239+
final int x = 2;
240+
241+
try (BufferedReader reader = FileUtil.getReader(&quot;D://INSERT.sql&quot;, StandardCharsets.UTF_8);
242+
BufferedWriter writer = FileUtil.getWriter(&quot;D://INSERT_&quot;+x+&quot;.sql&quot;, StandardCharsets.UTF_8, false)) &#123;
243+
244+
int n = 0;
245+
SqlParser sqlParser = new SqlParser(reader);
246+
while (true) &#123;
247+
String s = sqlParser.readOneSql();
248+
if (s == null || s.isEmpty()) &#123;
249+
break;
250+
&#125;
251+
if (n &lt; x) &#123;
252+
s = s.replace(&quot;;&quot;, &quot;,&quot;);
253+
&#125;
254+
if (n &gt; 0) &#123;
255+
s = s.substring(s.indexOf(&quot;VALUES&quot;) + 6);
256+
&#125;
257+
writer.append(s);
258+
if (n == x) &#123;
259+
n = 0;
260+
writer.append(&quot;\n&quot;);
261+
System.out.println(&quot;-&quot;);
262+
&#125; else &#123;
263+
n++;
264+
&#125;
265+
&#125;
266+
System.out.println(&quot;完成&quot;);
267+
&#125; catch (Exception e) &#123;
268+
e.printStackTrace();
269+
&#125;
270+
&#125;
271+
272+
273+
static class SqlParser &#123;
274+
private final Reader reader;
275+
276+
/**
277+
* 当前行号
278+
*/
279+
private long lineNo = -1;
280+
/**
281+
* 前一个特殊分界字符
282+
*/
283+
private int preChar = -1;
284+
285+
private final Buffer buf = new Buffer(IoUtil.DEFAULT_LARGE_BUFFER_SIZE);
286+
287+
288+
private boolean finished;
289+
/**
290+
* 当前读取SQL
291+
*/
292+
private final StrBuilder currentField = new StrBuilder(512);
293+
294+
SqlParser(Reader reader) &#123;
295+
this.reader = reader;
296+
&#125;
297+
298+
public String readOneSql() &#123;
299+
300+
final Buffer buf = this.buf;
301+
302+
int copyLen = 0; // 拷贝长度
303+
304+
// int preChar = this.preChar;//前一个特殊分界字符
305+
306+
final StrBuilder currentField = this.currentField;
307+
308+
boolean inComment = false;
309+
310+
while (true) &#123;
311+
if (false == buf.hasRemaining()) &#123;
312+
// 此Buffer读取结束，开始读取下一段
313+
if (copyLen &gt; 0) &#123;
314+
buf.appendTo(currentField, copyLen);
315+
// 此处无需mark，read方法会重置mark
316+
&#125;
317+
if (buf.read(this.reader) &lt; 0) &#123;
318+
// CSV读取结束
319+
finished = true;
320+
321+
if (currentField.hasContent() || preChar == &#39;;&#39;) &#123;
322+
// 剩余部分作为一个SQL返回
323+
return currentField.toStringAndReset();
324+
&#125;
325+
break;
326+
&#125;
327+
328+
// 重置
329+
copyLen = 0;
330+
&#125;
331+
332+
final char c = buf.get();
333+
334+
if (c == &#39;;&#39;) &#123;
335+
if (copyLen &gt; 0) &#123;
336+
buf.appendTo(currentField, copyLen);
337+
&#125;
338+
buf.mark();
339+
this.preChar = c;
340+
return currentField.append(c).toStringAndReset();
341+
&#125; else if (this.preChar == &#39;;&#39; &amp;&amp; (c == &#39;\r&#39; || c == &#39;\n&#39; || c == &#39; &#39;)) &#123;
342+
buf.mark();
343+
&#125; else &#123;
344+
copyLen++;
345+
this.preChar = c;
346+
&#125;
347+
348+
&#125;
349+
350+
return null;
351+
&#125;
352+
353+
&#125;
354+
355+
private static class Buffer implements Serializable &#123;
356+
private static final long serialVersionUID = 1L;
357+
358+
final char[] buf;
359+
360+
/**
361+
* 标记位置，用于读数据
362+
*/
363+
private int mark;
364+
/**
365+
* 当前位置
366+
*/
367+
private int position;
368+
/**
369+
* 读取的数据长度，一般小于buf.length，-1表示无数据
370+
*/
371+
private int limit;
372+
373+
Buffer(int capacity) &#123;
374+
buf = new char[capacity];
375+
&#125;
376+
377+
/**
378+
* 是否还有未读数据
379+
*
380+
* @return 是否还有未读数据
381+
*/
382+
public final boolean hasRemaining() &#123;
383+
return position &lt; limit;
384+
&#125;
385+
386+
/**
387+
* 读取到缓存&lt;br&gt;
388+
* 全量读取，会重置Buffer中所有数据
389+
*
390+
* @param reader &#123;@link Reader&#125;
391+
*/
392+
int read(Reader reader) &#123;
393+
int length;
394+
try &#123;
395+
length = reader.read(this.buf);
396+
&#125; catch (IOException e) &#123;
397+
throw new IORuntimeException(e);
398+
&#125;
399+
this.mark = 0;
400+
this.position = 0;
401+
this.limit = length;
402+
return length;
403+
&#125;
404+
405+
/**
406+
* 先获取当前字符，再将当前位置后移一位&lt;br&gt;
407+
* 此方法不检查是否到了数组末尾，请自行使用&#123;@link #hasRemaining()&#125;判断。
408+
*
409+
* @return 当前位置字符
410+
* @see #hasRemaining()
411+
*/
412+
char get() &#123;
413+
return this.buf[this.position++];
414+
&#125;
415+
416+
/**
417+
* 标记位置记为下次读取位置
418+
*/
419+
void mark() &#123;
420+
this.mark = this.position;
421+
&#125;
422+
423+
/**
424+
* 将数据追加到&#123;@link StrBuilder&#125;，追加结束后需手动调用&#123;@link #mark()&#125; 重置读取位置
425+
*
426+
* @param builder &#123;@link StrBuilder&#125;
427+
* @param length 追加的长度
428+
* @see #mark()
429+
*/
430+
void appendTo(StrBuilder builder, int length) &#123;
431+
builder.append(this.buf, this.mark, length);
432+
&#125;
433+
&#125;
434+
435+
&#125;
436+
</code></pre>
437+
<h1 id="二、执行结果"><a href="#二、执行结果" class="headerlink" title="二、执行结果"></a>二、执行结果</h1>
219438
</div>
220439

221440

‎archives/index.html

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ <h3>maven项目构建加速（一）命令参数篇</h3>
185185

186186
<span class="tag">
187187

188-
<a href="/tags/maven/" style="color: #00a596">maven</a>
188+
<a href="/tags/maven/" style="color: #03a9f4">maven</a>
189189
</span>
190190

191191
</span>
@@ -257,7 +257,7 @@ <h3>IntelliJ IDEA常用配置</h3>
257257

258258
<span class="tag">
259259

260-
<a href="/tags/IDEA/" style="color: #03a9f4">IDEA</a>
260+
<a href="/tags/IDEA/" style="color: #ffa2c4">IDEA</a>
261261
</span>
262262

263263
</span>
@@ -293,12 +293,12 @@ <h3>数据库批量Insert优化</h3>
293293

294294
<span class="tag">
295295

296-
<a href="/tags/SQL%E4%BC%98%E5%8C%96/" style="color: #03a9f4">SQL优化</a>
296+
<a href="/tags/SQL%E4%BC%98%E5%8C%96/" style="color: #ffa2c4">SQL优化</a>
297297
</span>
298298

299299
<span class="tag">
300300

301-
<a href="/tags/SQL%E5%90%88%E5%B9%B6/" style="color: #ff7d73">SQL合并</a>
301+
<a href="/tags/SQL%E5%90%88%E5%B9%B6/" style="color: #03a9f4">SQL合并</a>
302302
</span>
303303

304304
</span>

‎categories/IDEA配置/index.html

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ <h2>LOADING</h2>
169169

170170

171171
<span>
172-
<a href="/categories/%E6%95%99%E7%A8%8B/" style="background: #ffa2c4">
172+
<a href="/categories/%E6%95%99%E7%A8%8B/" style="background: #ff7d73">
173173
<span class="icon">
174174
<i class="fa-solid fa-bookmark fa-fw"></i>
175175
</span>
@@ -180,7 +180,7 @@ <h2>LOADING</h2>
180180

181181

182182
<span>
183-
<a href="/categories/SQL%E4%BC%98%E5%8C%96/" style="background: #ff7d73">
183+
<a href="/categories/SQL%E4%BC%98%E5%8C%96/" style="background: #ffa2c4">
184184
<span class="icon">
185185
<i class="fa-solid fa-bookmark fa-fw"></i>
186186
</span>
@@ -218,7 +218,7 @@ <h3>IntelliJ IDEA常用配置</h3>
218218

219219
<span class="tag">
220220

221-
<a href="/tags/IDEA/" style="color: #03a9f4">IDEA</a>
221+
<a href="/tags/IDEA/" style="color: #00a596">IDEA</a>
222222
</span>
223223

224224
</span>

‎categories/SQL优化/index.html

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ <h2>LOADING</h2>
156156

157157

158158
<span>
159-
<a href="/categories/IDEA%E9%85%8D%E7%BD%AE/" style="background: #ffa2c4">
159+
<a href="/categories/IDEA%E9%85%8D%E7%BD%AE/" style="background: #00a596">
160160
<span class="icon">
161161
<i class="fa-solid fa-bookmark fa-fw"></i>
162162
</span>
@@ -167,7 +167,7 @@ <h2>LOADING</h2>
167167

168168

169169
<span>
170-
<a href="/categories/%E6%95%99%E7%A8%8B/" style="background: #ff7d73">
170+
<a href="/categories/%E6%95%99%E7%A8%8B/" style="background: #00bcd4">
171171
<span class="icon">
172172
<i class="fa-solid fa-bookmark fa-fw"></i>
173173
</span>
@@ -223,7 +223,7 @@ <h3>数据库批量Insert优化</h3>
223223

224224
<span class="tag">
225225

226-
<a href="/tags/SQL%E5%90%88%E5%B9%B6/" style="color: #00a596">SQL合并</a>
226+
<a href="/tags/SQL%E5%90%88%E5%B9%B6/" style="color: #ffa2c4">SQL合并</a>
227227
</span>
228228

229229
</span>

‎categories/index.html

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ <h2>LOADING</h2>
156156

157157

158158
<span>
159-
<a href="/categories/IDEA%E9%85%8D%E7%BD%AE/" style="background: #03a9f4">
159+
<a href="/categories/IDEA%E9%85%8D%E7%BD%AE/" style="background: #00a596">
160160
<span class="icon">
161161
<i class="fa-solid fa-bookmark fa-fw"></i>
162162
</span>
@@ -167,7 +167,7 @@ <h2>LOADING</h2>
167167

168168

169169
<span>
170-
<a href="/categories/%E6%95%99%E7%A8%8B/" style="background: #ff7d73">
170+
<a href="/categories/%E6%95%99%E7%A8%8B/" style="background: #03a9f4">
171171
<span class="icon">
172172
<i class="fa-solid fa-bookmark fa-fw"></i>
173173
</span>
@@ -178,7 +178,7 @@ <h2>LOADING</h2>
178178

179179

180180
<span>
181-
<a href="/categories/SQL%E4%BC%98%E5%8C%96/" style="background: #ffa2c4">
181+
<a href="/categories/SQL%E4%BC%98%E5%8C%96/" style="background: #ff7d73">
182182
<span class="icon">
183183
<i class="fa-solid fa-bookmark fa-fw"></i>
184184
</span>

0 commit comments

Comments
 (0)