Page 1 of 2 12 LastLast
Results 1 to 10 of 14

Thread: Problems migrating to 2.0.3

  1. #1
    Join Date
    Jun 2008
    Location
    Argentina
    Posts
    9

    Default Problems migrating to 2.0.3

    We've migrating from Spring batch 1.1.1 to 2.0.3 and we are experiencing a problem behavior.
    We have some jobs that must run during the day, so they use a similar scheduling configuration.

    NOTE:All jobs are working fine if they don't share the scheduling windows.

    We are dealing with the folowing problem.
    the first Job (ValidadorJob) starts, and then starts procesing files, then if the Second jobs starts when the 1st step of the 1st job is working, the item reader (validadorStepItemProvider) gets into an infinite loop.

    This behavior is repeated if we change the start delay in the scheduler congifuration of the Second Job.
    For example, if we modify the start delay of the 2nd job calculating that the job is going to start when the other step of the first job is running (transformarStep), the same problem appears, TransformacionFaseItemProvider starts looping.


    Thanks in advance....
    Mario

    This is our jobs config.

    ********* SCHEDULE CONFIGURATION JOB ************************
    Code:
    <bean id="validadorTrigger" class="org.springframework.scheduling.quartz.SimpleTriggerBean">
    	<property name="jobDetail" ref="validadorLauncher" />
    	<property name="startDelay" value="2000" />
    	<property name="repeatInterval" value="600000" />
    	<property name="repeatCount" value="-1" />
    </bean>
    
    <bean id="importadorEstadoTrigger" class="org.springframework.scheduling.quartz.SimpleTriggerBean">
    	<property name="jobDetail" ref="importadorEstadoLauncher" />
    	<property name="startDelay" value="14000" />
    	<property name="repeatInterval" value="600000" />
    	<property name="repeatCount" value="-1" />
    </bean>
    
    <bean id="validadorLauncher" class="org.springframework.scheduling.quartz.JobDetailBean">
    	<property name="jobClass" value="com.swissmedical.rem.batch.launcher.FileScanQuartzBatchJobLauncher" />
    	<property name="jobDataAsMap">
    		<map>		
    			<entry key="batchContainer" 	value="batchContainer" 			/>
    			<entry key="hostName" 			value="localhost" 				/>
    			<entry key="jobName" 			value="validadorJob"      		/>
    			<entry key="jobStream" 			value="validatorStream" 		/>
    			<entry key="userName" 			value="${user.name}" 			/>
    			<entry key="jobRun" 			value="1" 						/>
    			<entry key="directoryName" 		value="${validadorJob.validadorStep.workingDirectory}"	/>
    		</map>
    	</property>
    </bean>
    
    <bean id="importadorEstadoLauncher" class="org.springframework.scheduling.quartz.JobDetailBean">
    	<property name="jobClass" value="com.swissmedical.rem.batch.launcher.FileScanQuartzBatchJobLauncher" />
    	<property name="jobDataAsMap">
    		<map>
    			<entry key="batchContainer" 	value="batchContainer" 			/>
    			<entry key="hostName" 			value="localhost" 				/>
    			<entry key="jobName" 			value="importadorEstadoJob"   	/>
    			<entry key="jobStream" 			value="importadorEstadoStream"  />
    			<entry key="userName" 			value="${user.name}" 			/>
    			<entry key="jobRun" 			value="1" 						/>
    			<entry key="directoryName" 		value="${importadorEstadoJob.importadorStep.workingDirectory}"	/>
    		</map>
    	</property>
    </bean>
    ************** FIRST JOB ***********************************
    Code:
    <job id="validadorJob" job-repository="simpleJobRepository" >
    	<step id="validarStep" next="moveFilesValidadorStep">
    		<tasklet transaction-manager="transactionManager">
    			<chunk reader="validadorStepItemProvider" processor="validadorStepItemProcess" writer="validadorStepItemWriter" commit-interval="1" skip-limit="10">
    				<skippable-exception-classes>
    					java.lang.Exception
    				</skippable-exception-classes>
    			</chunk>
    			<listeners>
    				<listener ref="stepListenerStep"/>
    				<listener ref="errorCreateFileListenerStep"/>
    			</listeners>
    		</tasklet>
    	</step>	
    
    ......
    OMITING other steps config (see validadorJob.xml)
    ......
    	<step id="transformarStep" next="moveFilesTransformadosStep">
    				<tasklet transaction-manager="transactionManager">
    					<chunk reader="transformadorStepItemProvider" processor="transformadorStepItemProcess" writer="transformadorStepItemWriter" commit-interval="1" skip-limit="10">
    						<skippable-exception-classes>
    							java.lang.Exception
    						</skippable-exception-classes>
    					</chunk>
    					<listeners>
    						<listener ref="stepListenerStep"/>
    						<listener ref="errorCreateFileTransfromacionListenerStep"/>
    					</listeners>
    				</tasklet>
    	</step>	
    		
    </job>
    
    <beans:bean id="validadorStepItemProvider" class="com.swissmedical.rem.batch.item.provider.DirectoryItemProvider">
    	<beans:property name="fileLister" ref="filesToValidateFileLister" />
    	<beans:property name="fileProcessor" ref="fileToValidateFileProcessor" />
    </beans:bean>
    
    <beans:bean id="validadorStepItemProcess" class="com.swissmedical.rem.batch.item.processor.ValidadorItemProcessor">
    	<beans:property name="validadorService" ref="validadorService" /> 
    	<beans:property name="erroresMostrar" ref="codeErrorToShowFileProcessor" />
    	<beans:property name="streamConverter" ref="ingresoFileToValidateStreamConverter" />
    	<beans:property name="errorFileProcessor" ref="afterErroneousProcessingFileRenamer" />	
    </beans:bean>
    
    <beans:bean id="validadorStepItemWriter" class="com.swissmedical.rem.batch.item.writer.ValidadorItemWriter">
    	<beans:property name="successFileProcessor" ref="afterSuccessfulProcessingFileRenamer" />
    </beans:bean>
    ********************* SECOND JOB **************************
    Code:
    <job id="importadorEstadoJob" job-repository="simpleJobRepository" >
    	<step id="importadorEstadoStep" next="moveFilesEstadoStep">
    		<tasklet transaction-manager="transactionManager">
    			<chunk reader="importadorEstadoStepItemProvider" processor="importadorEstadoItemProcessor" writer="importadorEstadoItemWriter" commit-interval="1" skip-limit="10">
    				<skippable-exception-classes>
    					java.lang.Exception
    				</skippable-exception-classes>
    			</chunk>
    			<listeners>
    				<listener ref="stepListenerStep"/>
    			</listeners>
    		</tasklet>
    	</step>	
    
    	<step id="moveFilesEstadoStep" parent="taskletStep">
    		<tasklet ref="moveFilesImportadorEstadoBean" transaction-manager="transactionManager"/>
    	</step>	
    	
    </job>
    
    <beans:bean id="stepListenerStep" class="com.swissmedical.rem.batch.interceptor.CleanUpFileAfterProcessingInterceptor">
    	<beans:property name="service" ref="errorImportacionService" />
    </beans:bean>
    
    <beans:bean id="importadorEstadoItemProcessor" class="com.swissmedical.rem.batch.item.processor.ImportadorEstadoItemProcessor">
    	<beans:property name="importadorEstadoService" ref="importadorEstadoService" />
    	<beans:property name="streamConverter" ref="estadoFileToValidateStreamConverter" />
    </beans:bean>
    
    <beans:bean id="importadorEstadoItemWriter" class="com.swissmedical.rem.batch.item.writer.ImportadorEstadoItemWriter">
    	<beans:property name="successFileProcessor" ref="afterSuccessfulProcessingFileRenamer" />
    </beans:bean>

  2. #2
    Join Date
    Dec 2006
    Posts
    1,061

    Default

    It's tough to say, since it I think you didn't include the second jobs reader, but given the first one is listed as a DirectoryItemProvider, which I'm assuming you wrote, my guess would be that it's something about your readers that can't be running at the same time. I'm not sure why this would just show up when upgrading though. Are you sure it's an infinite loop and not just hanging on the read? We really need some more information about exactly what's happening, and what the readers are doing to know whether it's framework related, or just your custom implementation.

  3. #3
    Join Date
    Jun 2008
    Location
    Argentina
    Posts
    9

    Default

    Thanks for replying quickly
    Here I'm attaching some configuration files and our custom iplementation of the items readers. I think that this would be enough to diagnosticate.
    Let me now if you need something else.

    Thanks again.
    Mario
    Attached Files Attached Files

  4. #4
    Join Date
    Jun 2008
    Location
    Argentina
    Posts
    9

    Default Jira

    Hi, Lucas,
    Is there any posibility that my problem could be related with this recent Jira ticket?

    http://forum.springsource.org/newrep...ote=1&p=260664

    Regards,
    Mario

  5. #5
    Join Date
    Jun 2008
    Location
    Argentina
    Posts
    9

    Default Keep trying on this

    Hi lucas,
    I developed another Job. And i'm facing with the same problem.
    I'm not using a custom reader this time.
    All that I could find out is that the job gets into an infinite loop when I try to skip an exception. So, if I avoid to use skipable exceptions or the job simply don't throw one. All works fine.
    Here is my Job configurartion.

    Code:
    
    	<bean id="headerRecordTokenizer" class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
    		<property name="delimiter" value=">"/>
    		<property name="names" value="nombreArchivo,md5Code" />
    	</bean>
    		
    	<beans:bean id="customRecordTokenizer" class="org.springframework.batch.item.file.transform.FixedLengthTokenizer">
    		<beans:property name="names" value="nombreArchivo,separador,md5Code" />
    		<beans:property name="columns" value="1-48,49-58,59-90" />
    	</beans:bean>
    
    	<beans:bean id="fieldSetMapper"  class="com.swissmedical.rem.batch.item.mapper.TxtRecordMD5Mapper" />
    Code:
    <job id="validadorMD5Job" job-repository="simpleJobRepository" >
    		<step id="validarMD5Step">
    			<tasklet transaction-manager="transactionManager" >
    				<chunk reader="validadorMD5StepItemReader" 
    					   processor="validadorMD5StepItemProcessor" 
    					   writer="validadorMD5StepItemWriter" 
    					   commit-interval="10"
    					   skip-limit="100">				
    					<streams>
    							<stream ref="validadorMD5StepItemReader"/>
    							<stream ref="validadorMD5StepItemWriter"/>
    					</streams>
    					<skippable-exception-classes>
    						java.io.FileNotFoundException
    					</skippable-exception-classes>
    				</chunk>
    				<listeners>
    					<listener ref="MD5HeaderAndFooterCallback"/>
    					<!--<listener ref="fileNameListener" />
    				--></listeners>					
    			</tasklet>
    			<!--<next on="COMPLETED WITH SKIPS" to="comunicarErrores" />
    			--><fail on="FAILED" exit-code="FAILED"/>
    		</step>
    
    		<!--<step id="comunicarErrores">
    			<tasklet ref="comunicarErroresRem"/>
    		</step>	
    	--></job>
    	
    	<beans:bean id="comunicarErroresRem" class="com.swissmedical.rem.batch.common.ErrorLogTasklet">
    		<!-- <beans:property name="dataSource" ref="dataSource"/> -->
    	</beans:bean>
    
    	<beans:bean id="itemReaderParent" class="org.springframework.batch.item.file.MultiResourceItemReader" abstract="true">
    		<beans:property name="delegate">
    			<beans:bean class="org.springframework.batch.item.file.FlatFileItemReader">
    				<!-- <beans:property name="resource" value="#{stepExecutionContext[fileName]}/*"/>  -->
    				<beans:property name="lineMapper">
    					<beans:bean class="org.springframework.batch.item.file.mapping.PatternMatchingCompositeLineMapper">
    						<beans:property name="tokenizers">
    							<beans:map>
    								<beans:entry key="rem*" value-ref="headerRecordTokenizer" />
    								<beans:entry key="documentos*" value-ref="customRecordTokenizer" />
    							</beans:map>
    						</beans:property>
    						<beans:property name="fieldSetMappers">
    							<beans:map>
    								<beans:entry key="rem*" value-ref="fieldSetMapper" />
    								<beans:entry key="documentos*" value-ref="fieldSetMapper" />
    							</beans:map>
    						</beans:property>
    					</beans:bean>
    				</beans:property>
    			</beans:bean>
    		</beans:property>
    	</beans:bean>
    
    	<beans:bean id="MD5HeaderAndFooterCallback" class="com.swissmedical.rem.batch.support.MD5StepSupport" />
    
    	<beans:bean id="validadorMD5StepItemReader" parent="itemReaderParent" scope="step" autowire-candidate="false" > 
    		 <beans:property name="resources" value="${validadorMD5Job.workingDirectory}/*.md5" />
    	</beans:bean>
    	
    	<beans:bean id="fileOutputLocator" class="org.springframework.core.io.FileSystemResource">
    		<beans:constructor-arg type="java.lang.String" value="${outPutMd5}" />
    	</beans:bean>
    	
    	<beans:bean id="validadorMD5StepItemProcessor" class="com.swissmedical.rem.batch.item.processor.ValidadorMD5ItemProcessor">
    	</beans:bean>
    	<!--
    	<beans:bean id="validadorMD5StepItemWriter" class="com.swissmedical.rem.batch.item.writer.ValidadorMD5ItemWriter">
    	  <beans:property name="successFileProcessor" ref="afterSuccessfulProcessingFileRenamer" /> 
    	</beans:bean>
    	
    
    	<beans:bean id="fileNameListener" class="com.swissmedical.rem.batch.common.OutputFileListener" />-->	
    	
    	<beans:bean id="validadorMD5StepItemWriter" class="org.springframework.batch.item.file.FlatFileItemWriter" scope="step">
    	    <beans:property name="resource" value="${outPutMd5}" />
    		<beans:property name="lineAggregator">
    			<beans:bean class="org.springframework.batch.item.file.transform.DelimitedLineAggregator">
    				<beans:property name="delimiter" value=">" />
    				<beans:property name="fieldExtractor">
    					<beans:bean class="org.springframework.batch.item.file.transform.BeanWrapperFieldExtractor">
    						<beans:property name="names" value="fileName,fileMD5,estado" />
    					</beans:bean>
    				</beans:property>
    			</beans:bean>
    		</beans:property>
    	</beans:bean>
    	<!--	
    	<beans:bean id="outputResource" class="org.springframework.core.io.FileSystemResource">
    		<beans:constructor-arg value="${outPutMd5}" />	
    	</beans:bean>	
    	-->

  6. #6
    Join Date
    Jun 2008
    Location
    Argentina
    Posts
    9

    Default

    I'm sorry I put a wrong link
    This is the correct one.
    Is related with an ItemWriter, however, i'm experiencing the same with a ItemReader

    http://jira.springframework.org/browse/BATCH-1354

    I'm using SB 2.0.3 were this problem (of the ItemWriter) was solved.

  7. #7
    Join Date
    Jun 2005
    Posts
    4,230

    Default

    Why does your reader/writer throw an Error? If it does then change that anyway. But maybe that's not the problem. There isn't enough detail yet. Probably you should post your reader / writer implementations.

  8. #8
    Join Date
    Jun 2005
    Posts
    4,230

    Default

    I missed the .zip attachment you provided earlier, sorry, that was useful. Isn't your business logic in a loop (I didn't follow it all exactly)? There is a file rename piece, and if your item craps out before it gets there then the file filter picks up that file again, and again, and again?

  9. #9
    Join Date
    Jun 2008
    Location
    Argentina
    Posts
    9

    Default Avoiding custom reader

    Hi Dave, thanks for trying too help me.
    At the begining I thought the same, for that reason I post other job without a custom reader implementation.
    Here is a job development with 2.1.0.M1. Without custom readers.

    The classes defined by me in this job are just, the Processor (Which thows the exception) , a Listener a FieldSetMapper a Pojo item to be process (File record).

    Here is the main job definition, I also attached the whole job and container definition, a couple of files, and a log file in case that you need them.

    Code:
    <job id="validadorMD5Job" job-repository="simpleJobRepository" >
    		<step id="validarMD5Step" >
    			<tasklet >
    				<chunk 	reader="validadorMD5StepItemReader" 
    						processor="validadorMD5StepItemProcessor" 
    						writer="validadorMD5StepItemWriter"						
    					   	commit-interval="50" 
    					   	skip-limit="10"
    					   >				
    					<streams>
    						<stream ref="validadorMD5StepItemReader"/>
    					</streams>
    					<skippable-exception-classes merge="true">
    						<include class="java.io.FileNotFoundException" />
    					</skippable-exception-classes>
    				</chunk>
    					<listeners>
    						<listener ref="comunicarErroresRem" />
    					</listeners>				
    			</tasklet>
    		</step>	
    	</job>
    Reader , writer processor and listener
    Code:
    <beans:bean id="comunicarErroresRem" class="com.swissmedical.rem.batch.item.listeners.ErrorLogListener">
    	</beans:bean>
    
    	<beans:bean id="itemReaderParent" class="org.springframework.batch.item.file.MultiResourceItemReader" abstract="true">
    		<beans:property name="delegate">
    			<beans:bean class="org.springframework.batch.item.file.FlatFileItemReader">
    				<beans:property name="lineMapper">
    					<beans:bean class="org.springframework.batch.item.file.mapping.PatternMatchingCompositeLineMapper">
    						<beans:property name="tokenizers">
    							<beans:map>
    								<beans:entry key="rem*" value-ref="headerRecordTokenizer" />
    								<beans:entry key="documentos*" value-ref="customRecordTokenizer" />
    							</beans:map>
    						</beans:property>
    						<beans:property name="fieldSetMappers">
    							<beans:map>
    								<beans:entry key="rem*" value-ref="fieldSetMapper" />
    								<beans:entry key="documentos*" value-ref="fieldSetMapper" />
    							</beans:map>
    						</beans:property>
    					</beans:bean>
    				</beans:property>
    			</beans:bean>
    		</beans:property>
    	</beans:bean>
    
    	<beans:bean id="validadorMD5StepItemReader" parent="itemReaderParent" scope="step" autowire-candidate="false" > 
    		 <beans:property name="resources" value="${validadorMD5Job.workingDirectory}/*.md5" />
    	</beans:bean>
    		
    	<beans:bean id="validadorMD5StepItemProcessor" class="com.swissmedical.rem.batch.item.processor.ValidadorMD5ItemProcessor">
    	</beans:bean>
    	
    	<beans:bean id="validadorMD5StepItemWriter" class="com.swissmedical.rem.batch.item.writer.ValidadorMD5ItemWriter">
    	</beans:bean>
    Mappers and tokenizers

    Code:
    	<beans:bean id="headerRecordTokenizer" class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
    		<beans:property name="delimiter" value=">"/>
    		<beans:property name="names" value="nombreArchivo,md5Code" />
    	</beans:bean>
    		
    	<beans:bean id="customRecordTokenizer" class="org.springframework.batch.item.file.transform.FixedLengthTokenizer">
    		<beans:property name="names" value="nombreArchivo,separador,md5Code" />
    		<beans:property name="columns" value="1-48,49-58,59-90" />
    	</beans:bean>
    	
    	<beans:bean id="fieldSetMapper"  class="com.swissmedical.rem.batch.item.mapper.md5.TxtRecordMD5Mapper" />
    If you check the log file you will see that the process always ends when the skip limit is reached. (It doesn't matter if it's 1 or 1000...)

    Note: Nowadays I'm using a FileNotFoundException just to try to solve this, but the idea is to use a ProccesingException (Cutom Exception implementation) for all the jobs to log the errors against a service

  10. #10
    Join Date
    Jun 2008
    Location
    Argentina
    Posts
    9

    Default adding zip file

    I'm sorry here is the zip file
    Attached Files Attached Files

Posting Permissions

  • You may not post new threads
  • You may not post replies
  • You may not post attachments
  • You may not edit your posts
  •