Results 1 to 9 of 9

Thread: Multiple output files in Spring batch

  1. #1
    Join Date
    Apr 2010
    Location
    Bangalore
    Posts
    43

    Default Multiple output files in Spring batch

    Hi,
    I am very new to spring batch. I have a requirement like this, I have 100 records from the database, I want the first 10 records to be in 1 file, and the next 10 records, i.e 11-20 in other file and so on.. I searched the net, but was unable to get anything, I searched this forum, and got this answer in one of the thread..

    "This is supported, however, there isn't any kind of 'max record count' that would signal to the FileOutputSource that a new file should be created. The outputsource would simply need to be closed and reopened. However, a wrapper could easily be written that would do this for you".

    But, I am not able to do this wrapper and understand what it means. Can anyone please help me out in this..

  2. #2
    Join Date
    Dec 2005
    Location
    Lyon, France
    Posts
    311

    Default

    I think the MultiResourceItemWriter is exactly what you need.

  3. #3
    Join Date
    Apr 2010
    Location
    Bangalore
    Posts
    43

    Default

    Thanks Arno for the reply.. I am very new to spring Batch, so ,can you please explain, how to use it, or can you provide me link explaining this.

  4. #4
    Join Date
    Dec 2005
    Location
    Lyon, France
    Posts
    311

    Default

    the MultiResourceItemWriter is just a wrapper: it's gonna delegate the writing to a FlatFileItemWriter for example, count the number of chunks, and route the writing to new files if needed. A basic usage would be:
    Code:
    <bean id="writer" class="org.springframework.batch.item.file.MultiResourceItemWriter">
      <property name="delegate" ref="realWriter" />
      <property name="itemCountLimitPerResource" value="10" />
      <property name="resource" value="file:/output/output.txt" />
    </bean>
    The created files would be output.txt.1, output.txt.2, etc (you can customize this with the resourceSuffixCreator property).

    For more information, have a look at the Javadoc of MultiResourceItemWriter.

  5. #5
    Join Date
    Apr 2010
    Location
    Bangalore
    Posts
    43

    Default

    Thanks Arno, I have implemented it, but still i get all the records in a single output file.. I have commit-interval to 2 and itemCountLimitPerResource also 2..
    Can, you please help me, pointing to where i am going wrong. my job configuration file is..
    Code:
    <?xml version="1.0" encoding="UTF-8"?>
    <beans xmlns="http://www.springframework.org/schema/beans"
    	xmlns:aop="http://www.springframework.org/schema/aop" xmlns:tx="http://www.springframework.org/schema/tx"
    	xmlns:p="http://www.springframework.org/schema/p" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    	xsi:schemaLocation="
    		http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
    		http://www.springframework.org/schema/batch http://www.springframework.org/schema/batch/spring-batch-2.0.xsd
    		http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
    		http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx.xsd">
    	
    	<import resource="../Memory-JobRepository.xml"/>
    	
    
    	<job id="headerFooterSample" xmlns="http://www.springframework.org/schema/batch">
    		<step id="step1">
    			<tasklet>
    				<chunk reader="reader" writer="writer" commit-interval="2">
    					<streams>
    						<stream ref="reader"/>
    						<stream ref="mywriter"/>
    					</streams>
    				</chunk>
    				<listeners>
    					<listener ref="footerCallback"/>
    					<listener ref="headerCopier"/>
    				</listeners>
    			</tasklet>
    		</step>
    	</job>
    
    	<bean id="reader" class="org.springframework.batch.item.file.FlatFileItemReader">
    		<property name="resource" ref="inputResource" />
    		<property name="lineMapper">
    			<bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
    				<property name="lineTokenizer">
    					<bean
    						class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
    						<property name="delimiter" value="," />
    					</bean>
    				</property>
    				<property name="fieldSetMapper">
    					<bean
    						class="org.springframework.batch.item.file.mapping.PassThroughFieldSetMapper" />
    				</property>
    			</bean>
    		</property>
    		<property name="skippedLinesCallback" ref="headerCopier" />
    		<property name="linesToSkip" value="1" />
    	</bean>
    	<bean id="writer" class="org.springframework.batch.item.file.FlatFileItemWriter" scope="step">
    		<!-- <property name="resource" ref="outputResource" /> -->
    		
    		<property name="lineAggregator">
    			<!-- <bean
    				class="org.springframework.batch.item.file.transform.PassThroughLineAggregator" />-->
    			<bean class="org.springframework.batch.sample.support.MyLineAggregator" />
    		</property>
    		<property name="headerCallback" ref="headerCopier" />
    		<property name="footerCallback" ref="footerCallback" />
    	</bean>
    	<bean id="footerCallback" class="org.springframework.batch.sample.support.SummaryFooterCallback" />
    	<bean id="headerCopier"
    		class="org.springframework.batch.sample.support.HeaderCopyCallback" />
    	<bean id="inputResource" class="org.springframework.core.io.ClassPathResource">
    		<constructor-arg type="java.lang.String"
    			value="data/input.txt" />
    	</bean>
    	<bean id="outputResource" class="org.springframework.core.io.FileSystemResource" scope="step">
    			 <!-- <constructor-arg type="java.lang.String"
    			value="target/test-outputs/headerFooterOutput.txt" /> -->
    			<constructor-arg value="#{jobParameters[file.name]}.dat"/>
    	</bean>
    	
    	<bean id="mywriter" class="org.springframework.batch.item.file.MultiResourceItemWriter">
      		<property name="delegate" ref="writer" />
      		<property name="itemCountLimitPerResource" value="2" />
      	<property name="resource" ref="outputResource" />
    </bean>
    	
    </beans>
    My input file contains 10 lines..

  6. #6
    Join Date
    Dec 2005
    Location
    Lyon, France
    Posts
    311

    Default

    in the chunk tag, you refer to the writer bean, which is the delegate. Refer to the mywriter bean instead.

  7. #7
    Join Date
    Apr 2010
    Location
    Bangalore
    Posts
    43

    Default

    Thanks Arno, It is now working.. but what happens is, if i have 10 lines in file and commit-interval=2.. i get 5 files, fine, but the file2,3,4 will be empty and file 1 has header and 2 lines.. 5th file is having footer only..

  8. #8
    Join Date
    Dec 2005
    Location
    Lyon, France
    Posts
    311

    Default

    this behavior is explained in the Javadoc. I think it would make more sense with longer files :-)

  9. #9
    Join Date
    Jun 2012
    Posts
    1

    Default

    Hi All,

    I am trying to do a similar thing along with writing the output also to a database. I have a large input XML file whose data i want to write Simultaneously to database and also to 10 files (each containing a fixed set of records .. File 1 contaning first 100 records, file 2 containing next 10 records and so on )

    When i just try writing to 10 files using the configuration mentioned here .. it does work .. however when i try using a compositeitemwriter to have data base writing i get an error -


    SEVERE: Encountered an error executing the step
    java.lang.ClassCastException: org.springframework.batch.item.file.MultiResourceI temWriter cannot be cast to org.springframework.batch.item.file.FlatFileItemWr iter
    at com.cts.clinicaltransformation.poc.SeparatorCompos iteItemWriter.write(SeparatorCompositeItemWriter.j ava:33)
    at org.springframework.batch.core.step.item.SimpleChu nkProcessor.writeItems(SimpleChunkProcessor.java:1 71)
    at org.springframework.batch.core.step.item.SimpleChu nkProcessor.doWrite(SimpleChunkProcessor.java:150)
    at org.springframework.batch.core.step.item.SimpleChu nkProcessor.write(SimpleChunkProcessor.java:269)
    at org.springframework.batch.core.step.item.SimpleChu nkProcessor.process(SimpleChunkProcessor.java:194)
    at org.springframework.batch.core.step.item.ChunkOrie ntedTasklet.execute(ChunkOrientedTasklet.java:74)
    at org.springframework.batch.core.step.tasklet.Taskle tStep$ChunkTransactionCallback.doInTransaction(Tas kletStep.java:386)
    at org.springframework.transaction.support.Transactio nTemplate.execute(TransactionTemplate.java:128)
    at org.springframework.batch.core.step.tasklet.Taskle tStep$2.doInChunkContext(TaskletStep.java:264)
    at org.springframework.batch.core.scope.context.StepC ontextRepeatCallback.doInIteration(StepContextRepe atCallback.java:76)
    at org.springframework.batch.repeat.support.RepeatTem plate.getNextResult(RepeatTemplate.java:367)
    at org.springframework.batch.repeat.support.RepeatTem plate.executeInternal(RepeatTemplate.java:214)
    at org.springframework.batch.repeat.support.RepeatTem plate.iterate(RepeatTemplate.java:143)
    at org.springframework.batch.core.step.tasklet.Taskle tStep.doExecute(TaskletStep.java:250)
    at org.springframework.batch.core.step.AbstractStep.e xecute(AbstractStep.java:195)
    at org.springframework.batch.core.job.SimpleStepHandl er.handleStep(SimpleStepHandler.java:135)
    at org.springframework.batch.core.job.flow.JobFlowExe cutor.executeStep(JobFlowExecutor.java:61)
    at org.springframework.batch.core.job.flow.support.st ate.StepState.handle(StepState.java:60)
    at org.springframework.batch.core.job.flow.support.Si mpleFlow.resume(SimpleFlow.java:144)
    at org.springframework.batch.core.job.flow.support.Si mpleFlow.start(SimpleFlow.java:124)
    at org.springframework.batch.core.job.flow.FlowJob.do Execute(FlowJob.java:135)
    at org.springframework.batch.core.job.AbstractJob.exe cute(AbstractJob.java:281)
    at org.springframework.batch.core.launch.support.Simp leJobLauncher$1.run(SimpleJobLauncher.java:120)
    at org.springframework.core.task.SyncTaskExecutor.exe cute(SyncTaskExecutor.java:49)
    at org.springframework.batch.core.launch.support.Simp leJobLauncher.run(SimpleJobLauncher.java:114)
    at org.springframework.batch.core.launch.support.Comm andLineJobRunner.start(CommandLineJobRunner.java:3 49)
    at org.springframework.batch.core.launch.support.Comm andLineJobRunner.main(CommandLineJobRunner.java:57 4)
    Jun 6, 2012 4:10:13 PM org.springframework.batch.core.launch.support.Simp leJobLauncher$1 run
    INFO: Job: [FlowJob: [name=compositeItemWriterJob]] completed with the following parameters: [{}] and the following status: [FAILED]
    Jun 6, 2012 4:10:13 PM org.springframework.context.support.AbstractApplic ationCont

    Please help me with an relevant example

Tags for this Thread

Posting Permissions

  • You may not post new threads
  • You may not post replies
  • You may not post attachments
  • You may not edit your posts
  •