107 	xfs_buf_t	*bp,  in xfs_buf_item_flush_log_debug()  argument
114 	bip = XFS_BUF_FSPRIVATE(bp, xfs_buf_log_item_t*);  in xfs_buf_item_flush_log_debug()
140 	xfs_buf_t	*bp;  in xfs_buf_item_log_check()  local
145 	bp = bip->bli_buf;  in xfs_buf_item_log_check()
146 	ASSERT(XFS_BUF_COUNT(bp) > 0);  in xfs_buf_item_log_check()
147 	ASSERT(XFS_BUF_PTR(bp) != NULL);  in xfs_buf_item_log_check()
149 	buffer = XFS_BUF_PTR(bp);  in xfs_buf_item_log_check()
150 	for (x = 0; x < XFS_BUF_COUNT(bp); x++) {  in xfs_buf_item_log_check()
154 				bip, bp, orig, x);  in xfs_buf_item_log_check()
162 STATIC void	xfs_buf_error_relse(xfs_buf_t *bp);
163 STATIC void	xfs_buf_do_callbacks(xfs_buf_t *bp, xfs_log_item_t *lip);
182 	xfs_buf_t	*bp;  in xfs_buf_item_size()  local
196 	bp = bip->bli_buf;  in xfs_buf_item_size()
223 		} else if (xfs_buf_offset(bp, next_bit * XFS_BLI_CHUNK) !=  in xfs_buf_item_size()
224 			   (xfs_buf_offset(bp, last_bit * XFS_BLI_CHUNK) +  in xfs_buf_item_size()
251 	xfs_buf_t	*bp;  in xfs_buf_item_format()  local
261 	bp = bip->bli_buf;  in xfs_buf_item_format()
262 	ASSERT(XFS_BUF_BP_ISMAPPED(bp));  in xfs_buf_item_format()
321 			vecp->i_addr = xfs_buf_offset(bp, buffer_offset);  in xfs_buf_item_format()
327 			vecp->i_addr = xfs_buf_offset(bp, buffer_offset);  in xfs_buf_item_format()
334 		} else if (xfs_buf_offset(bp, next_bit << XFS_BLI_SHIFT) !=  in xfs_buf_item_format()
335 			   (xfs_buf_offset(bp, last_bit << XFS_BLI_SHIFT) +  in xfs_buf_item_format()
338 			vecp->i_addr = xfs_buf_offset(bp, buffer_offset);  in xfs_buf_item_format()
372 	xfs_buf_t	*bp;  in xfs_buf_item_pin()  local
374 	bp = bip->bli_buf;  in xfs_buf_item_pin()
375 	ASSERT(XFS_BUF_ISBUSY(bp));  in xfs_buf_item_pin()
380 	xfs_buftrace("XFS_PIN", bp);  in xfs_buf_item_pin()
381 	xfs_bpin(bp);  in xfs_buf_item_pin()
400 	xfs_buf_t	*bp;  in xfs_buf_item_unpin()  local
404 	bp = bip->bli_buf;  in xfs_buf_item_unpin()
405 	ASSERT(bp != NULL);  in xfs_buf_item_unpin()
406 	ASSERT(XFS_BUF_FSPRIVATE(bp, xfs_buf_log_item_t *) == bip);  in xfs_buf_item_unpin()
409 	xfs_buftrace("XFS_UNPIN", bp);  in xfs_buf_item_unpin()
413 	xfs_bunpin(bp);  in xfs_buf_item_unpin()
416 		ASSERT(XFS_BUF_VALUSEMA(bp) <= 0);  in xfs_buf_item_unpin()
417 		ASSERT(!(XFS_BUF_ISDELAYWRITE(bp)));  in xfs_buf_item_unpin()
418 		ASSERT(XFS_BUF_ISSTALE(bp));  in xfs_buf_item_unpin()
421 		xfs_buftrace("XFS_UNPIN STALE", bp);  in xfs_buf_item_unpin()
429 			xfs_buf_do_callbacks(bp, (xfs_log_item_t *)bip);  in xfs_buf_item_unpin()
430 			XFS_BUF_SET_FSPRIVATE(bp, NULL);  in xfs_buf_item_unpin()
431 			XFS_BUF_CLR_IODONE_FUNC(bp);  in xfs_buf_item_unpin()
435 			xfs_buf_item_relse(bp);  in xfs_buf_item_unpin()
436 			ASSERT(XFS_BUF_FSPRIVATE(bp, void *) == NULL);  in xfs_buf_item_unpin()
438 		xfs_buf_relse(bp);  in xfs_buf_item_unpin()
454 	xfs_buf_t		*bp;  in xfs_buf_item_unpin_remove()  local
458 	bp = bip->bli_buf;  in xfs_buf_item_unpin_remove()
466 		xfs_buftrace("XFS_UNPIN_REMOVE", bp);  in xfs_buf_item_unpin_remove()
482 		XFS_BUF_SET_FSPRIVATE2(bp, NULL);  in xfs_buf_item_unpin_remove()
500 	xfs_buf_t	*bp;  in xfs_buf_item_trylock()  local
502 	bp = bip->bli_buf;  in xfs_buf_item_trylock()
504 	if (XFS_BUF_ISPINNED(bp)) {  in xfs_buf_item_trylock()
508 	if (!XFS_BUF_CPSEMA(bp)) {  in xfs_buf_item_trylock()
517 	XFS_BUF_HOLD(bp);  in xfs_buf_item_trylock()
545 	xfs_buf_t	*bp;  in xfs_buf_item_unlock()  local
548 	bp = bip->bli_buf;  in xfs_buf_item_unlock()
549 	xfs_buftrace("XFS_UNLOCK", bp);  in xfs_buf_item_unlock()
554 	XFS_BUF_SET_FSPRIVATE2(bp, NULL);  in xfs_buf_item_unlock()
607 		xfs_buf_item_relse(bp);  in xfs_buf_item_unlock()
616 		xfs_buf_relse(bp);  in xfs_buf_item_unlock()
661 	xfs_buf_t	*bp;  in xfs_buf_item_abort()  local
663 	bp = bip->bli_buf;  in xfs_buf_item_abort()
664 	xfs_buftrace("XFS_ABORT", bp);  in xfs_buf_item_abort()
665 	XFS_BUF_SUPER_STALE(bp);  in xfs_buf_item_abort()
681 	xfs_buf_t	*bp;  in xfs_buf_item_push()  local
686 	bp = bip->bli_buf;  in xfs_buf_item_push()
688 	if (XFS_BUF_ISDELAYWRITE(bp)) {  in xfs_buf_item_push()
689 		xfs_bawrite(bip->bli_item.li_mountp, bp);  in xfs_buf_item_push()
691 		xfs_buf_relse(bp);  in xfs_buf_item_push()
733 	xfs_buf_t	*bp,  in xfs_buf_item_init()  argument
747 	if (XFS_BUF_FSPRIVATE3(bp, xfs_mount_t *) != mp)  in xfs_buf_item_init()
748 		XFS_BUF_SET_FSPRIVATE3(bp, mp);  in xfs_buf_item_init()
749 	XFS_BUF_SET_BDSTRAT_FUNC(bp, xfs_bdstrat_cb);  in xfs_buf_item_init()
750 	if (XFS_BUF_FSPRIVATE(bp, void *) != NULL) {  in xfs_buf_item_init()
751 		lip = XFS_BUF_FSPRIVATE(bp, xfs_log_item_t *);  in xfs_buf_item_init()
763 	chunks = (int)((XFS_BUF_COUNT(bp) + (XFS_BLI_CHUNK - 1)) >> XFS_BLI_SHIFT);  in xfs_buf_item_init()
771 	bip->bli_buf = bp;  in xfs_buf_item_init()
773 	bip->bli_format.blf_blkno = (__int64_t)XFS_BUF_ADDR(bp);  in xfs_buf_item_init()
774 	bip->bli_format.blf_len = (ushort)BTOBB(XFS_BUF_COUNT(bp));  in xfs_buf_item_init()
789 	bip->bli_orig = (char *)kmem_alloc(XFS_BUF_COUNT(bp), KM_SLEEP);  in xfs_buf_item_init()
790 	memcpy(bip->bli_orig, XFS_BUF_PTR(bp), XFS_BUF_COUNT(bp));  in xfs_buf_item_init()
791 	bip->bli_logged = (char *)kmem_zalloc(XFS_BUF_COUNT(bp) / NBBY, KM_SLEEP);  in xfs_buf_item_init()
798 	if (XFS_BUF_FSPRIVATE(bp, void *) != NULL) {  in xfs_buf_item_init()
800 				XFS_BUF_FSPRIVATE(bp, xfs_log_item_t *);  in xfs_buf_item_init()
802 	XFS_BUF_SET_FSPRIVATE(bp, bip);  in xfs_buf_item_init()
916 	xfs_buf_t	*bp)  in xfs_buf_item_relse()  argument
920 	xfs_buftrace("XFS_RELSE", bp);  in xfs_buf_item_relse()
921 	bip = XFS_BUF_FSPRIVATE(bp, xfs_buf_log_item_t*);  in xfs_buf_item_relse()
922 	XFS_BUF_SET_FSPRIVATE(bp, bip->bli_item.li_bio_list);  in xfs_buf_item_relse()
923 	if ((XFS_BUF_FSPRIVATE(bp, void *) == NULL) &&  in xfs_buf_item_relse()
924 	    (XFS_BUF_IODONE_FUNC(bp) != NULL)) {  in xfs_buf_item_relse()
925 		ASSERT((XFS_BUF_ISUNINITIAL(bp)) == 0);  in xfs_buf_item_relse()
926 		XFS_BUF_CLR_IODONE_FUNC(bp);  in xfs_buf_item_relse()
930 	kmem_free(bip->bli_orig, XFS_BUF_COUNT(bp));  in xfs_buf_item_relse()
932 	kmem_free(bip->bli_logged, XFS_BUF_COUNT(bp) / NBBY);  in xfs_buf_item_relse()
954 	xfs_buf_t	*bp,  in xfs_buf_attach_iodone()  argument
960 	ASSERT(XFS_BUF_ISBUSY(bp));  in xfs_buf_attach_iodone()
961 	ASSERT(XFS_BUF_VALUSEMA(bp) <= 0);  in xfs_buf_attach_iodone()
964 	if (XFS_BUF_FSPRIVATE(bp, void *) != NULL) {  in xfs_buf_attach_iodone()
965 		head_lip = XFS_BUF_FSPRIVATE(bp, xfs_log_item_t *);  in xfs_buf_attach_iodone()
969 		XFS_BUF_SET_FSPRIVATE(bp, lip);  in xfs_buf_attach_iodone()
972 	ASSERT((XFS_BUF_IODONE_FUNC(bp) == xfs_buf_iodone_callbacks) ||  in xfs_buf_attach_iodone()
973 	       (XFS_BUF_IODONE_FUNC(bp) == NULL));  in xfs_buf_attach_iodone()
974 	XFS_BUF_SET_IODONE_FUNC(bp, xfs_buf_iodone_callbacks);  in xfs_buf_attach_iodone()
979 	xfs_buf_t	*bp,  in xfs_buf_do_callbacks()  argument
994 		lip->li_cb(bp, lip);  in xfs_buf_do_callbacks()
1008 	xfs_buf_t	*bp)  in xfs_buf_iodone_callbacks()  argument
1015 	ASSERT(XFS_BUF_FSPRIVATE(bp, void *) != NULL);  in xfs_buf_iodone_callbacks()
1016 	lip = XFS_BUF_FSPRIVATE(bp, xfs_log_item_t *);  in xfs_buf_iodone_callbacks()
1018 	if (XFS_BUF_GETERROR(bp) != 0) {  in xfs_buf_iodone_callbacks()
1026 			ASSERT(XFS_BUF_TARGET(bp) == mp->m_ddev_targp);  in xfs_buf_iodone_callbacks()
1027 			XFS_BUF_SUPER_STALE(bp);  in xfs_buf_iodone_callbacks()
1028 			xfs_buftrace("BUF_IODONE_CB", bp);  in xfs_buf_iodone_callbacks()
1029 			xfs_buf_do_callbacks(bp, lip);  in xfs_buf_iodone_callbacks()
1030 			XFS_BUF_SET_FSPRIVATE(bp, NULL);  in xfs_buf_iodone_callbacks()
1031 			XFS_BUF_CLR_IODONE_FUNC(bp);  in xfs_buf_iodone_callbacks()
1040 			if (XFS_BUF_ISSHUT(bp)) {  in xfs_buf_iodone_callbacks()
1041 			    XFS_BUF_UNSHUT(bp);  in xfs_buf_iodone_callbacks()
1042 				xfs_buf_relse(bp);  in xfs_buf_iodone_callbacks()
1044 				xfs_biodone(bp);  in xfs_buf_iodone_callbacks()
1050 		if ((XFS_BUF_TARGET(bp) != lasttarg) ||  in xfs_buf_iodone_callbacks()
1055 			      XFS_BUF_TARGET(bp),  in xfs_buf_iodone_callbacks()
1056 			      (__uint64_t)XFS_BUF_ADDR(bp), mp->m_fsname);  in xfs_buf_iodone_callbacks()
1058 		lasttarg = XFS_BUF_TARGET(bp);  in xfs_buf_iodone_callbacks()
1060 		if (XFS_BUF_ISASYNC(bp)) {  in xfs_buf_iodone_callbacks()
1070 			XFS_BUF_ERROR(bp,0); /* errno of 0 unsets the flag */  in xfs_buf_iodone_callbacks()
1072 			if (!(XFS_BUF_ISSTALE(bp))) {  in xfs_buf_iodone_callbacks()
1073 				XFS_BUF_DELAYWRITE(bp);  in xfs_buf_iodone_callbacks()
1074 				XFS_BUF_DONE(bp);  in xfs_buf_iodone_callbacks()
1075 				XFS_BUF_SET_START(bp);  in xfs_buf_iodone_callbacks()
1077 			ASSERT(XFS_BUF_IODONE_FUNC(bp));  in xfs_buf_iodone_callbacks()
1078 			xfs_buftrace("BUF_IODONE ASYNC", bp);  in xfs_buf_iodone_callbacks()
1079 			xfs_buf_relse(bp);  in xfs_buf_iodone_callbacks()
1095 			XFS_BUF_SET_BRELSE_FUNC(bp,xfs_buf_error_relse);  in xfs_buf_iodone_callbacks()
1096 			XFS_BUF_DONE(bp);  in xfs_buf_iodone_callbacks()
1097 			XFS_BUF_V_IODONESEMA(bp);  in xfs_buf_iodone_callbacks()
1102 	xfs_buftrace("XFS BUFCB NOERR", bp);  in xfs_buf_iodone_callbacks()
1104 	xfs_buf_do_callbacks(bp, lip);  in xfs_buf_iodone_callbacks()
1105 	XFS_BUF_SET_FSPRIVATE(bp, NULL);  in xfs_buf_iodone_callbacks()
1106 	XFS_BUF_CLR_IODONE_FUNC(bp);  in xfs_buf_iodone_callbacks()
1107 	xfs_biodone(bp);  in xfs_buf_iodone_callbacks()
1116 	xfs_buf_t	*bp)  in xfs_buf_error_relse()  argument
1121 	lip = XFS_BUF_FSPRIVATE(bp, xfs_log_item_t *);  in xfs_buf_error_relse()
1123 	ASSERT(XFS_BUF_TARGET(bp) == mp->m_ddev_targp);  in xfs_buf_error_relse()
1125 	XFS_BUF_STALE(bp);  in xfs_buf_error_relse()
1126 	XFS_BUF_DONE(bp);  in xfs_buf_error_relse()
1127 	XFS_BUF_UNDELAYWRITE(bp);  in xfs_buf_error_relse()
1128 	XFS_BUF_ERROR(bp,0);  in xfs_buf_error_relse()
1129 	xfs_buftrace("BUF_ERROR_RELSE", bp);  in xfs_buf_error_relse()
1136 	xfs_buf_do_callbacks(bp, lip);  in xfs_buf_error_relse()
1137 	XFS_BUF_SET_FSPRIVATE(bp, NULL);  in xfs_buf_error_relse()
1138 	XFS_BUF_CLR_IODONE_FUNC(bp);  in xfs_buf_error_relse()
1139 	XFS_BUF_SET_BRELSE_FUNC(bp,NULL);  in xfs_buf_error_relse()
1140 	xfs_buf_relse(bp);  in xfs_buf_error_relse()
1154 	xfs_buf_t		*bp,  in xfs_buf_iodone()  argument
1160 	ASSERT(bip->bli_buf == bp);  in xfs_buf_iodone()
1180 	kmem_free(bip->bli_orig, XFS_BUF_COUNT(bp));  in xfs_buf_iodone()
1182 	kmem_free(bip->bli_logged, XFS_BUF_COUNT(bp) / NBBY);  in xfs_buf_iodone()
1198 	xfs_buf_t		*bp;  in xfs_buf_item_trace()  local
1201 	bp = bip->bli_buf;  in xfs_buf_item_trace()
1209 				(0xFFFFFFFF & XFS_BUF_ADDR(bp) >> 32)),  in xfs_buf_item_trace()
1210 		     (void *)((unsigned long)(0xFFFFFFFF & XFS_BUF_ADDR(bp))),  in xfs_buf_item_trace()
1211 		     (void *)((unsigned long)XFS_BUF_COUNT(bp)),  in xfs_buf_item_trace()
1212 		     (void *)((unsigned long)XFS_BUF_BFLAGS(bp)),  in xfs_buf_item_trace()
1213 		     XFS_BUF_FSPRIVATE(bp, void *),  in xfs_buf_item_trace()
1214 		     XFS_BUF_FSPRIVATE2(bp, void *),  in xfs_buf_item_trace()
1215 		     (void *)(unsigned long)XFS_BUF_ISPINNED(bp),  in xfs_buf_item_trace()
1216 		     (void *)XFS_BUF_IODONE_FUNC(bp),  in xfs_buf_item_trace()
1217 		     (void *)((unsigned long)(XFS_BUF_VALUSEMA(bp))),  in xfs_buf_item_trace()