Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

New parameters to fix encoding problem of out files and to speed up schema ddl exports #46

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

New parameters to fix encoding problem of out files and to speed up schema ddl exports #46

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
efb672f
An option for the encoding of output files added.
yorukoglu Nov 25, 2016
3b5cfe7
To speedup schema ddl exports, last ddl time related filters added.
yorukoglu Nov 28, 2016
e2aef36
README.md file updated for new parameters added.
yorukoglu Nov 28, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ More command line options
-url, DB connection URL
example: scott/tiger@localhost:1521:ORCL
-o, --output, output dir
-oe, --out-encoding, out file encoding
example: UTF-8
-p, --parallel, number of parallel thread (default 4)
-s, --schemas, a comma separated list of schemas for processing
(works only if connected to oracle as sysdba)
Expand All @@ -51,6 +53,14 @@ More command line options
every LIKE wildcard can be used
-tf, --type-filter, filter for specific DDL object types
-tfm, --type-filtermode, mode for type filter: include(default) or exclude
-dta, --ddl-time-after, export objects with last DDL time after the time specified
example: 2016-11-27 : exports objects that modified after 27 Nov 2016
example: "2016-11-27 01:00:00" : exports objects that modified after 27 Nov 2016, 1am
-dtb, --ddl-time-before, export objects with last DDL time before the time specified
example: "2016-11-27 01:00:00"
-dti, --ddl-time-in, export objects with last DDL time in last n[minute|hour|day]
example: "6 hour" : exports objects that modified in last 6 hours
example: 2d : exports objects that modified in last 2 days
--stop-on-warning, stop on getting DDL error (skip by default)
-rsv, replace actual sequence values with 1
--replace-sequence-values,
Expand Down
117 changes: 116 additions & 1 deletion src/main/java/com/googlecode/scheme2ddl/Main.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
* @author A_Reshetnikov
Expand All @@ -26,6 +33,7 @@ public class Main {

private static final Log log = LogFactory.getLog(Main.class);
public static String outputPath = null;
public static String outputEncoding = null;
public static int parallelCount = 4;
private static boolean justPrintUsage = false;
private static boolean justPrintVersion = false;
Expand All @@ -42,6 +50,11 @@ public class Main {
private static String objectFilter = "%";
private static String typeFilter = "";
private static String typeFilterMode = "include";
private static String ddlTimeAfter = null ;
private static String ddlTimeBefore = null ;
private static String ddlTimeIn = null ;
private static DateFormat dfDateTime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
private static DateFormat dfDate = new SimpleDateFormat("yyyy-MM-dd");

public static void main(String[] args) throws Exception {
typeFilterMode = "include"; //default is to include any type filter
Expand All @@ -68,7 +81,13 @@ public static void main(String[] args) throws Exception {
System.out.println("DDL object filter: " + objectFilter);
System.out.println("DDL type filter: " + typeFilter);
System.out.println("DDL type filter mode: " + typeFilterMode);
new UserObjectJobRunner().start(context, isLaunchedByDBA, objectFilter.toLowerCase(), typeFilter.toUpperCase(), typeFilterMode.toLowerCase());
if (ddlTimeAfter != null)
System.out.println("Last DDL time after: " + ddlTimeAfter);
if (ddlTimeBefore != null)
System.out.println("Last DDL time before: " + ddlTimeBefore);
if (ddlTimeIn != null)
System.out.println("DDL time in last : " + ddlTimeIn);
new UserObjectJobRunner().start(context, isLaunchedByDBA, objectFilter.toLowerCase(), typeFilter.toUpperCase(), typeFilterMode.toLowerCase(), ddlTimeAfter, ddlTimeBefore, ddlTimeIn);
}
}

Expand Down Expand Up @@ -97,6 +116,10 @@ private static void modifyContext(ConfigurableApplicationContext context) {
UserObjectWriter writer = (UserObjectWriter) context.getBean("writer");
writer.setOutputPath(outputPath);
}
if (outputEncoding != null) {
UserObjectWriter writer = (UserObjectWriter) context.getBean("writer");
writer.setFileEncoding(outputEncoding);
}
if (parallelCount > 0) {
SimpleAsyncTaskExecutor taskExecutor = (SimpleAsyncTaskExecutor) context.getBean("taskExecutor");
taskExecutor.setConcurrencyLimit(parallelCount);
Expand All @@ -107,6 +130,32 @@ private static void modifyContext(ConfigurableApplicationContext context) {
ConnectionDao connectionDao = (ConnectionDao) context.getBean("connectionDao");
isLaunchedByDBA = connectionDao.hasSelectCatalogRole(); //todo rename isLaunchedByDBA -> processForeignSchema
}

Map<String,Object> ddlTimeFilterMap = (Map<String,Object>)context.getBean("ddlTimeFilterMap");
if (ddlTimeAfter == null && ddlTimeFilterMap != null && ddlTimeFilterMap instanceof Map && ddlTimeFilterMap.containsKey("ddlTimeAfter")){
String da = (String)ddlTimeFilterMap.get("ddlTimeAfter");
if (da != null)
ddlTimeAfter = da;
}
if (ddlTimeBefore == null && ddlTimeFilterMap != null && ddlTimeFilterMap instanceof Map && ddlTimeFilterMap.containsKey("ddlTimeBefore")){
String df = (String)ddlTimeFilterMap.get("ddlTimeBefore");
if (df != null)
ddlTimeBefore = df;
}
if (ddlTimeIn == null && ddlTimeFilterMap != null && ddlTimeFilterMap instanceof Map && ddlTimeFilterMap.containsKey("ddlTimeIn")){
Map<String,String> dim = (Map<String,String>) ddlTimeFilterMap.get("ddlTimeIn");
if (dim != null && dim instanceof Map && dim.containsKey("n") && dim.containsKey("unit")){
String unit = dim.get("unit");
String n = dim.get("n");

if ("day".equals(unit))
ddlTimeIn = n + " /*day*/";
if ("hour".equals(unit))
ddlTimeIn = n + " /*hour*/ /24";
if ("minute".equals(unit))
ddlTimeIn = n + " /*minute*/ /(24*60)";
}
}
//process schemas
processSchemas(context);

Expand Down Expand Up @@ -250,6 +299,8 @@ private static void printUsage() {
msg.append(" example: scott/tiger@localhost:1521:ORCL" + lSep);

msg.append(" -o, --output, output dir" + lSep);
msg.append(" -oe, --out-encoding, out file encoding" + lSep);
msg.append(" example: UTF-8" + lSep);
msg.append(" -p, --parallel, number of parallel thread (default 4)" + lSep);
msg.append(" -s, --schemas, a comma separated list of schemas for processing" + lSep);
msg.append(" (works only if connected to oracle as sysdba)" + lSep);
Expand All @@ -258,6 +309,14 @@ private static void printUsage() {
msg.append(" every LIKE wildcard can be used" + lSep);
msg.append(" -tf, --type-filter, filter for specific DDL object types" + lSep);
msg.append(" -tfm, --type-filtermode, mode for type filter: include(default) or exclude" + lSep);
msg.append(" -dta, --ddl-time-after, export objects with last DDL time after the time specified" + lSep);
msg.append(" example: 2016-11-27 : exports objects that modified after 27 Nov 2016" + lSep);
msg.append(" example: \"2016-11-27 01:00:00\" : exports objects that modified after 27 Nov 2016, 1am" + lSep);
msg.append(" -dtb, --ddl-time-before, export objects with last DDL time before the time specified" + lSep);
msg.append(" example: \"2016-11-27 01:00:00\"" + lSep);
msg.append(" -dti, --ddl-time-in, export objects with last DDL time in last n[minute|hour|day]" + lSep);
msg.append(" example: \"6 hour\" : exports objects that modified in last 6 hours" + lSep);
msg.append(" example: 2d : exports objects that modified in last 2 days" + lSep);
msg.append(" --stop-on-warning, stop on getting DDL error (skip by default)" + lSep);
msg.append(" -rsv, replace actual sequence values with 1 " + lSep);
msg.append(" --replace-sequence-values, " + lSep);
Expand Down Expand Up @@ -295,6 +354,9 @@ private static void collectArgs(String[] args) throws Exception {
} else if (arg.equals("-o") || arg.equals("-output") || arg.equals("--output")) {
outputPath = args[i + 1];
i++;
} else if (arg.equals("-oe") || arg.equals("-out-encoding") || arg.equals("--out-encoding")) {
outputEncoding = args[i + 1];
i++;
} else if (arg.equals("-s") || arg.equals("-schemas") || arg.equals("--schemas")) {
schemas = args[i + 1];
i++;
Expand All @@ -303,6 +365,59 @@ private static void collectArgs(String[] args) throws Exception {
i++;
} else if (arg.equals("-tc") || arg.equals("--test-connection")) {
justTestConnection = true;
} else if (arg.equals("-dta") || arg.equals("--ddl-time-after") || arg.equals("-ddl-time-after")) {
ddlTimeAfter = args[i + 1];
i++;
Date d = null;
try{
d = dfDateTime.parse(ddlTimeAfter);
ddlTimeAfter = dfDateTime.format(d);
}catch(ParseException pe){
try{
d = dfDate.parse(ddlTimeAfter);
ddlTimeAfter = dfDateTime.format(d);
}catch(ParseException pe2){
System.err.println("Invalid parameter format for " + arg);
printUsage();
throw new Exception("");
}
}
} else if (arg.equals("-dtb") || arg.equals("--ddl-time-before") || arg.equals("-ddl-time-before")) {
ddlTimeBefore = args[i + 1];
i++;
Date d = null;
try{
d = dfDateTime.parse(ddlTimeBefore);
ddlTimeBefore = dfDateTime.format(d);
}catch(ParseException pe){
try{
d = dfDate.parse(ddlTimeBefore);
ddlTimeBefore = dfDateTime.format(d);
}catch(ParseException pe2){
System.err.println("Invalid parameter format for " + arg);
printUsage();
throw new Exception("");
}
}
} else if (arg.equals("-dti") || arg.equals("--ddl-time-in") || arg.equals("-ddl-time-in")) {
String param = args[i + 1];
i++;
Pattern pattern = Pattern.compile("^(\\d+)\\s?(minute|min|m|hour|h|day|d)?");
Matcher m = pattern.matcher(param);
if (m.find()){
String unit = m.group(2);
String n = m.group(1);
if ("day".equals(unit) || "d".equals(unit) || unit == null)
ddlTimeIn = n + " /*day*/ ";
if ("hour".equals(unit) || "h".equals(unit))
ddlTimeIn = n + " /*hour*/ /24";
if ("minute".equals(unit) || "min".equals(unit) || "m".equals(unit))
ddlTimeIn = n + " /*minute*/ /(24*60)";
}else{
System.err.println("Invalid parameter format for " + arg);
printUsage();
throw new Exception("");
}
} else if (arg.equals("--stop-on-warning")) {
stopOnWarning = true;
} else if ((arg.equals("-rsv") || arg.equals("--replace-sequence-values"))) {
Expand Down
5 changes: 4 additions & 1 deletion src/main/java/com/googlecode/scheme2ddl/UserObjectJobRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public class UserObjectJobRunner {
protected static final Log logger = LogFactory.getLog(UserObjectJobRunner.class);
private JobLauncher launcher;

int start(ConfigurableApplicationContext context, boolean launchedByDBA, String objectFilter, String typeFilter, String typeFilterMode) throws Exception {
int start(ConfigurableApplicationContext context, boolean launchedByDBA, String objectFilter, String typeFilter, String typeFilterMode, String ddlTimeAfter, String ddlTimeBefore, String ddlTimeIn) throws Exception {
try {
context.getAutowireCapableBeanFactory().autowireBeanProperties(this,
AutowireCapableBeanFactory.AUTOWIRE_BY_TYPE, false);
Expand All @@ -40,6 +40,9 @@ int start(ConfigurableApplicationContext context, boolean launchedByDBA, String
parametersBuilder.addString("objectFilter", objectFilter);
parametersBuilder.addString("typeFilter", typeFilter);
parametersBuilder.addString("typeFilterMode", typeFilterMode);
parametersBuilder.addString("ddlTimeAfter", ddlTimeAfter);
parametersBuilder.addString("ddlTimeBefore", ddlTimeBefore);
parametersBuilder.addString("ddlTimeIn", ddlTimeIn);
JobParameters jobParameters = parametersBuilder.toJobParameters();
logger.trace(String.format("Start spring batch job with parameters %s", jobParameters));
JobExecution jobExecution = launcher.run(job, jobParameters);
Expand Down
10 changes: 9 additions & 1 deletion src/main/java/com/googlecode/scheme2ddl/UserObjectWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ public class UserObjectWriter implements ItemWriter<UserObject> {

private static final Log log = LogFactory.getLog(UserObjectWriter.class);
private String outputPath;
private String fileEncoding;


public void write(List<? extends UserObject> data) throws Exception {
Expand All @@ -31,7 +32,10 @@ public void writeUserObject(UserObject userObject) throws Exception {
String absoluteFileName = outputPath + "/" + userObject.getFileName();
absoluteFileName = FilenameUtils.separatorsToSystem(absoluteFileName);
File file = new File(absoluteFileName);
FileUtils.writeStringToFile(file, userObject.getDdl());
if(fileEncoding==null)
FileUtils.writeStringToFile(file, userObject.getDdl());
else
FileUtils.writeStringToFile(file, userObject.getDdl(), fileEncoding);
log.info(String.format("Saved %s %s.%s to file %s",
userObject.getType().toLowerCase(),
userObject.getSchema().toLowerCase(),
Expand All @@ -44,6 +48,10 @@ public void writeUserObject(UserObject userObject) throws Exception {
public void setOutputPath(String outputPath) {
this.outputPath = outputPath;
}

public void setFileEncoding(String fileEncoding) {
this.fileEncoding = fileEncoding;
}

@Deprecated
public void setFileNameCase(String fileNameCase) {
Expand Down
18 changes: 18 additions & 0 deletions src/main/java/com/googlecode/scheme2ddl/dao/UserObjectDaoImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ public class UserObjectDaoImpl extends JdbcDaoSupport implements UserObjectDao {
private String typeFilter;
@Value("#{jobParameters['typeFilterMode']}")
private String typeFilterMode = "include";
@Value("#{jobParameters['ddlTimeAfter']}")
private String ddlTimeAfter = null;
@Value("#{jobParameters['ddlTimeBefore']}")
private String ddlTimeBefore = null;
@Value("#{jobParameters['ddlTimeIn']}")
private String ddlTimeIn = null;

public List<UserObject> findListForProccessing() {
String sql;
Expand All @@ -56,6 +62,12 @@ public List<UserObject> findListForProccessing() {

sql += " IN (" + typeFilter + ") ";
}
if (ddlTimeAfter != null)
sql += " and t.last_ddl_time >= TO_DATE('" + ddlTimeAfter + "', 'YYYY-MM-DD HH24:MI:SS') ";
if (ddlTimeBefore != null)
sql += " and t.last_ddl_time <= TO_DATE('" + ddlTimeBefore + "', 'YYYY-MM-DD HH24:MI:SS') ";
if (ddlTimeIn != null)
sql += " and t.last_ddl_time >= SYSDATE-" + ddlTimeIn + " ";
if (isTypeAllowed("'REFRESH GROUP'")) {
sql += " UNION ALL " +
" select rname as object_name, 'REFRESH_GROUP' as object_type " +
Expand All @@ -79,6 +91,12 @@ public List<UserObject> findListForProccessing() {

sql += " IN (" + typeFilter + ") ";
}
if (ddlTimeAfter != null)
sql += " and t.last_ddl_time >= TO_DATE('" + ddlTimeAfter + "', 'YYYY-MM-DD HH24:MI:SS') ";
if (ddlTimeBefore != null)
sql += " and t.last_ddl_time <= TO_DATE('" + ddlTimeBefore + "', 'YYYY-MM-DD HH24:MI:SS') ";
if (ddlTimeIn != null)
sql += " and t.last_ddl_time >= SYSDATE-" + ddlTimeIn + " ";
if (isTypeAllowed("'REFRESH GROUP'")) {
sql += " UNION ALL " +
" select rname as object_name, 'REFRESH_GROUP' as object_type " +
Expand Down
17 changes: 17 additions & 0 deletions src/main/resources/scheme2ddl.config.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,22 @@
<!--<value>SCOTT</value>-->
<!--<value>HR</value>-->
</util:list>

<util:map id="ddlTimeFilterMap">

<!-- format for ddlTimeAfter and ddlTimeBefore : YYYY-MM-DD HH:MM:SS -->
<!-- entry key="ddlTimeAfter" value="2016-11-27 00:00:00"/>
<entry key="ddlTimeBefore" value="2016-11-28 06:00:00"/ -->

<!-- unit can be one of (minute, hour, day) -->
<!-- export ddl of objects that modified in last 2 days -->
<!-- entry key="ddlTimeIn">
<util:map>
<entry key="n" value="2"/>
<entry key="unit" value="day"/>
</util:map>
</entry -->
</util:map>

<bean id="reader" class="com.googlecode.scheme2ddl.UserObjectReader" scope="step">
<property name="processPublicDbLinks" value="true"/>
Expand All @@ -31,6 +47,7 @@

<bean id="writer" class="com.googlecode.scheme2ddl.UserObjectWriter">
<property name="outputPath" value="output"/>
<!-- <property name="fileEncoding" value="UTF-8"/> -->
</bean>


Expand Down